Skip to content

Switch WQP default profile from legacy to WQX3.0 #230

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
thodson-usgs wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Switch WQP default profile from legacy to WQX3.0 #230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 52 additions & 37 deletions dataretrieval/wqp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@

def get_results(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Query the WQP for results.
Expand Down Expand Up @@ -160,7 +160,7 @@ def get_results(

def what_sites(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Search WQP for sites within a region with specific data.
Expand Down Expand Up @@ -215,7 +215,7 @@ def what_sites(

def what_organizations(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Search WQP for organizations within a region with specific data.
Expand Down Expand Up @@ -258,8 +258,12 @@ def what_organizations(
if legacy is True:
url = wqp_url("Organization")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("Organization")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("Organization", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -268,7 +272,7 @@ def what_organizations(
return df, WQP_Metadata(response)


def what_projects(ssl_check=True, legacy=True, **kwargs):
def what_projects(ssl_check=True, legacy=False, **kwargs):
"""Search WQP for projects within a region with specific data.

Any WQP API parameter can be passed as a keyword argument to this function.
Expand Down Expand Up @@ -309,8 +313,12 @@ def what_projects(ssl_check=True, legacy=True, **kwargs):
if legacy is True:
url = wqp_url("Project")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("Project")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("Project", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -321,7 +329,7 @@ def what_projects(ssl_check=True, legacy=True, **kwargs):

def what_activities(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Search WQP for activities within a region with specific data.
Expand Down Expand Up @@ -385,7 +393,7 @@ def what_activities(

def what_detection_limits(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Search WQP for result detection limits within a region with specific
Expand Down Expand Up @@ -435,8 +443,12 @@ def what_detection_limits(
if legacy is True:
url = wqp_url("ResultDetectionQuantitationLimit")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("ResultDetectionQuantitationLimit")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("ResultDetectionQuantitationLimit", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -447,7 +459,7 @@ def what_detection_limits(

def what_habitat_metrics(
ssl_check=True,
legacy=True,
legacy=False,
**kwargs,
) -> tuple[DataFrame, WQP_Metadata]:
"""Search WQP for habitat metrics within a region with specific data.
Expand Down Expand Up @@ -490,8 +502,12 @@ def what_habitat_metrics(
if legacy is True:
url = wqp_url("BiologicalMetric")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("BiologicalMetric")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("BiologicalMetric", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -500,7 +516,7 @@ def what_habitat_metrics(
return df, WQP_Metadata(response)


def what_project_weights(ssl_check=True, legacy=True, **kwargs):
def what_project_weights(ssl_check=True, legacy=False, **kwargs):
"""Search WQP for project weights within a region with specific data.

Any WQP API parameter can be passed as a keyword argument to this function.
Expand Down Expand Up @@ -546,8 +562,12 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs):
if legacy is True:
url = wqp_url("ProjectMonitoringLocationWeighting")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("ProjectMonitoringLocationWeighting")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("ProjectMonitoringLocationWeighting", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -556,7 +576,7 @@ def what_project_weights(ssl_check=True, legacy=True, **kwargs):
return df, WQP_Metadata(response)


def what_activity_metrics(ssl_check=True, legacy=True, **kwargs):
def what_activity_metrics(ssl_check=True, legacy=False, **kwargs):
"""Search WQP for activity metrics within a region with specific data.

Any WQP API parameter can be passed as a keyword argument to this function.
Expand Down Expand Up @@ -602,8 +622,12 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs):
if legacy is True:
url = wqp_url("ActivityMetric")
else:
print("WQX3.0 profile not available, returning legacy profile.")
url = wqp_url("ActivityMetric")
warnings.warn(
"No WQX3.0 equivalent exists for this function; returning legacy data.",
UserWarning,
stacklevel=2,
)
url = wqp_url("ActivityMetric", warn=False)

response = query(url, payload=kwargs, delimiter=";", ssl_check=ssl_check)

Expand All @@ -612,11 +636,12 @@ def what_activity_metrics(ssl_check=True, legacy=True, **kwargs):
return df, WQP_Metadata(response)


def wqp_url(service):
def wqp_url(service, warn=True):
"""Construct the WQP URL for a given service."""

base_url = "https://www.waterqualitydata.us/data/"
_warn_legacy_use()
if warn:
_warn_legacy_use()

if service not in services_legacy:
raise TypeError(
Expand All @@ -631,7 +656,6 @@ def wqx3_url(service):
"""Construct the WQP URL for a given WQX 3.0 service."""

base_url = "https://www.waterqualitydata.us/wqx3/"
_warn_wqx3_use()

if service not in services_wqx3:
raise TypeError(
Expand Down Expand Up @@ -705,20 +729,11 @@ def _check_kwargs(kwargs):
return kwargs


def _warn_wqx3_use():
message = (
"Support for the WQX3.0 profiles is experimental. "
"Queries may be slow or fail intermitttently."
)
warnings.warn(message, UserWarning, stacklevel=2)


def _warn_legacy_use():
message = (
"This function call will return the legacy WQX format, "
"which means USGS data have not been updated since March 2024. "
"Please review the dataretrieval-python documentation for more "
"information on updated WQX3.0 profiles. Setting `legacy=False` "
"will remove this warning."
"You are explicitly requesting the legacy WQX format "
"(legacy=True), which means USGS data have not been updated "
"since March 2024. Remove legacy=True from your call to use "
"the current WQX3.0 profiles."
)
warnings.warn(message, DeprecationWarning, stacklevel=2)
46 changes: 22 additions & 24 deletions tests/wqp_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@


def test_get_results(requests_mock):
"""Tests water quality portal ratings query"""
"""Tests water quality portal results query using the default WQX3.0 profile"""
request_url = (
"https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762"
"https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762"
"&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011"
"&mimeType=csv"
"&dataProfile=fullPhysChem"
)
response_file_path = "tests/data/wqp_results.txt"
response_file_path = "tests/data/wqp3_results.txt"
mock_request(requests_mock, request_url, response_file_path)
df, md = get_results(
siteid="WIDNR_WQX-10032762",
Expand All @@ -33,42 +34,39 @@ def test_get_results(requests_mock):
startDateHi="09-30-2011",
)
assert type(df) is DataFrame
assert df.size == 315
assert df.size == 900
assert md.url == request_url
assert isinstance(md.query_time, datetime.timedelta)
assert md.header == {"mock_header": "value"}
assert md.comment is None


def test_get_results_WQX3(requests_mock):
"""Tests water quality portal results query with new WQX3.0 profile"""
def test_get_results_legacy(requests_mock):
"""Tests water quality portal results query with explicit legacy=True"""
request_url = (
"https://www.waterqualitydata.us/wqx3/Result/search?siteid=WIDNR_WQX-10032762"
"https://www.waterqualitydata.us/data/Result/Search?siteid=WIDNR_WQX-10032762"
"&characteristicName=Specific+conductance&startDateLo=05-01-2011&startDateHi=09-30-2011"
"&mimeType=csv"
"&dataProfile=fullPhysChem"
)
response_file_path = "tests/data/wqp3_results.txt"
response_file_path = "tests/data/wqp_results.txt"
mock_request(requests_mock, request_url, response_file_path)
df, md = get_results(
legacy=False,
siteid="WIDNR_WQX-10032762",
characteristicName="Specific conductance",
startDateLo="05-01-2011",
startDateHi="09-30-2011",
)
with pytest.warns(DeprecationWarning, match="legacy=True"):
df, md = get_results(
legacy=True,
siteid="WIDNR_WQX-10032762",
characteristicName="Specific conductance",
startDateLo="05-01-2011",
startDateHi="09-30-2011",
)
assert type(df) is DataFrame
assert df.size == 900
assert df.size == 315
assert md.url == request_url
assert isinstance(md.query_time, datetime.timedelta)
assert md.header == {"mock_header": "value"}
assert md.comment is None


def test_what_sites(requests_mock):
"""Tests Water quality portal sites query"""
"""Tests Water quality portal sites query using the default WQX3.0 profile"""
request_url = (
"https://www.waterqualitydata.us/data/Station/Search?statecode=US%3A34&characteristicName=Chloride"
"https://www.waterqualitydata.us/wqx3/Station/search?statecode=US%3A34&characteristicName=Chloride"
"&mimeType=csv"
)
response_file_path = "tests/data/wqp_sites.txt"
Expand Down Expand Up @@ -117,9 +115,9 @@ def test_what_projects(requests_mock):


def test_what_activities(requests_mock):
"""Tests Water quality portal activities query"""
"""Tests Water quality portal activities query using the default WQX3.0 profile"""
request_url = (
"https://www.waterqualitydata.us/data/Activity/Search?statecode=US%3A34&characteristicName=Chloride"
"https://www.waterqualitydata.us/wqx3/Activity/search?statecode=US%3A34&characteristicName=Chloride"
"&mimeType=csv"
)
response_file_path = "tests/data/wqp_activities.txt"
Expand Down
Loading