diff --git a/docs/sphinx/source/whatsnew/v0.16.0.rst b/docs/sphinx/source/whatsnew/v0.16.0.rst index 7f56a2cb30..f62b5af3a9 100644 --- a/docs/sphinx/source/whatsnew/v0.16.0.rst +++ b/docs/sphinx/source/whatsnew/v0.16.0.rst @@ -2,6 +2,8 @@ v0.16.0 +------- + Breaking Changes ~~~~~~~~~~~~~~~~ * Change output type of :py:func:`pvlib.irradiance.perez` and @@ -32,6 +34,9 @@ Bug fixes ``dc_ohmic_model="dc_ohms_from_percent"`` is used with a single-Array system and the weather/irradiance input is passed as a length-1 list or tuple. (:issue:`2829`) +* Fix :py:func:`~pvlib.iotools.get_merra2` to work with a new API endpoint + now that the previous one is retired. Format of returned data and metadata + is slightly different. (:pull:`2839`) Enhancements @@ -39,6 +44,9 @@ Enhancements * Map beam horizontal irradiance to ``bhi`` when :py:func:`~pvlib.iotools.get_era5` is called with ``map_variables=True``. (:pull:`2819`) +* Allow variables from multiple datasets to be requested at once in + :py:func:`~pvlib.iotools.get_merra2`. (:pull:`2839`) + Documentation ~~~~~~~~~~~~~ @@ -65,3 +73,4 @@ Contributors * Carolina Crespo (:ghuser:`cbcrespo`) * Andrew Chen (:ghuser:`chuenchen309`) * Sai Asish Y (:ghuser:`SAY-5`) +* Kevin Anderson (:ghuser:`kandersolar`) diff --git a/pvlib/iotools/merra2.py b/pvlib/iotools/merra2.py index 8a7770b9f4..975024499b 100644 --- a/pvlib/iotools/merra2.py +++ b/pvlib/iotools/merra2.py @@ -1,4 +1,5 @@ import pandas as pd +import numpy as np import requests from io import StringIO @@ -8,12 +9,20 @@ 'SWGDNCLR': 'ghi_clear', 'ALBEDO': 'albedo', 'LWGNT': 'longwave_net', - 'LWGEM': 'longwave_up', 'LWGAB': 'longwave_down', 'T2M': 'temp_air', - 'T2MDEW': 'temp_dew', 'PS': 'pressure', 'TOTEXTTAU': 'aod550', + 'TQV': 'precipitable_water', +} + + +def _k_to_c(temp_k): + return temp_k - 273.15 + + +UNITS = { + 'T2M': _k_to_c, } @@ -45,8 +54,11 @@ def get_merra2(latitude, longitude, start, end, username, password, dataset, NASA EarthData username. password : str NASA EarthData password. - dataset : str - Dataset name (with version), e.g. "M2T1NXRAD.5.12.4". + dataset : str or list of str + Dataset name (with version), e.g. "M2T1NXRAD.5.12.4". If all + variables are in the same dataset, this can be a single string. + Otherwise, pass a list of dataset names corresponding to + the list of requested variables. variables : list of str List of variable names to retrieve. See the documentation of the specific dataset you are accessing for options. @@ -70,45 +82,42 @@ def get_merra2(latitude, longitude, start, end, username, password, dataset, ----- The following datasets provide quantities useful for PV modeling: - +------------------------------------+-----------+---------------+ - | Dataset | Variable | pvlib name | - +====================================+===========+===============+ - | `M2T1NXRAD.5.12.4 `_ | SWGDN | ghi | - | +-----------+---------------+ - | | SWGDNCLR | ghi_clear | - | +-----------+---------------+ - | | ALBEDO | albedo | - | +-----------+---------------+ - | | LWGAB | longwave_down | - | +-----------+---------------+ - | | LWGNT | longwave_net | - | +-----------+---------------+ - | | LWGEM | longwave_up | - +------------------------------------+-----------+---------------+ - | `M2T1NXSLV.5.12.4 `_ | T2M | temp_air | - | +-----------+---------------+ - | | U10 | n/a | - | +-----------+---------------+ - | | V10 | n/a | - | +-----------+---------------+ - | | T2MDEW | temp_dew | - | +-----------+---------------+ - | | PS | pressure | - | +-----------+---------------+ - | | TO3 | n/a | - | +-----------+---------------+ - | | TQV | n/a | - +------------------------------------+-----------+---------------+ - | `M2T1NXAER.5.12.4 `_ | TOTEXTTAU | aod550 | - | +-----------+---------------+ - | | TOTSCATAU | n/a | - | +-----------+---------------+ - | | TOTANGSTR | n/a | - +------------------------------------+-----------+---------------+ + +------------------------------------+-----------+--------------------+ + | Dataset | Variable | pvlib name | + +====================================+===========+====================+ + | `M2T1NXRAD.5.12.4 `_ | SWGDN | ghi | + | +-----------+--------------------+ + | | SWGDNCLR | ghi_clear | + | +-----------+--------------------+ + | | ALBEDO | albedo | + | +-----------+--------------------+ + | | LWGNT | longwave_net | + +------------------------------------+-----------+--------------------+ + | `M2T1NXLFO.5.12.4 `_ | LWGAB | longwave_down | + +------------------------------------+-----------+--------------------+ + | `M2T1NXSLV.5.12.4 `_ | T2M | temp_air | + | +-----------+--------------------+ + | | U10M | n/a | + | +-----------+--------------------+ + | | V10M | n/a | + | +-----------+--------------------+ + | | PS | pressure | + | +-----------+--------------------+ + | | TO3 | n/a | + | +-----------+--------------------+ + | | TQV | precipitable_water | + +------------------------------------+-----------+--------------------+ + | `M2T1NXAER.5.12.4 `_ | TOTEXTTAU | aod550 | + | +-----------+--------------------+ + | | TOTSCATAU | n/a | + | +-----------+--------------------+ + | | TOTANGSTR | n/a | + +------------------------------------+-----------+--------------------+ .. _M2T1NXRAD: https://disc.gsfc.nasa.gov/datasets/M2T1NXRAD_5.12.4/summary .. _M2T1NXSLV: https://disc.gsfc.nasa.gov/datasets/M2T1NXSLV_5.12.4/summary .. _M2T1NXAER: https://disc.gsfc.nasa.gov/datasets/M2T1NXAER_5.12.4/summary + .. _M2T1NXLFO: https://disc.gsfc.nasa.gov/datasets/M2T1NXLFO_5.12.4/summary A complete list of datasets and their documentation is available at [3]_. @@ -121,9 +130,6 @@ def get_merra2(latitude, longitude, start, end, username, password, dataset, .. [3] https://disc.gsfc.nasa.gov/datasets?project=MERRA-2 """ - # general API info here: - # https://docs.unidata.ucar.edu/tds/5.0/userguide/netcdf_subset_service_ref.html # noqa: E501 - def _to_utc_dt_notz(dt): dt = pd.to_datetime(dt) if dt.tzinfo is not None: @@ -134,63 +140,68 @@ def _to_utc_dt_notz(dt): start = _to_utc_dt_notz(start) end = _to_utc_dt_notz(end) - if (year := start.year) != end.year: - raise ValueError("start and end must be in the same year (in UTC)") - - url = ( - "https://goldsmr4.gesdisc.eosdis.nasa.gov/thredds/ncss/grid/" - f"MERRA2_aggregation/{dataset}/{dataset}_Aggregation_{year}.ncml" + # login + login_url = "https://urs.earthdata.nasa.gov/api/users/find_or_create_token" + response = requests.post( + login_url, + auth=(username, password), + headers={"Accept": "application/json"}, + timeout=10, ) + response.raise_for_status() + token = response.json()["access_token"] + # data query + if isinstance(dataset, str): + datasets = [dataset] * len(variables) + else: + datasets = dataset + + data_url = "https://api.giovanni.earthdata.nasa.gov/timeseries" parameters = { - 'var': ",".join(variables), - 'latitude': latitude, - 'longitude': longitude, - 'time_start': start.isoformat() + "Z", - 'time_end': end.isoformat() + "Z", - 'accept': 'csv', + "location": "[{},{}]".format(round(latitude, 4), round(longitude, 4)), + "time": "{}/{}".format(start.isoformat(), end.isoformat()) } + query_headers = { + 'Authorization': f'Bearer {token}' + } + meta = {'dataset': dataset} + data = {} + for variable, dataset in zip(variables, datasets): + name = dataset.replace(".", "_") + "_" + variable + query_parameters = parameters.copy() + query_parameters["data"] = name - auth = (username, password) - - with requests.Session() as session: - session.auth = auth - login = session.request('get', url, params=parameters) - response = session.get(login.url, auth=auth, params=parameters) - - response.raise_for_status() - - content = response.content.decode('utf-8') - buffer = StringIO(content) - df = pd.read_csv(buffer) + response = requests.get(data_url, params=query_parameters, + headers=query_headers) + response.raise_for_status() + buffer = StringIO(response.text) - df.index = pd.to_datetime(df['time']) + var_meta = {} + while (line := buffer.readline().rstrip()) != "": + key, value = line.split(",", maxsplit=1) + var_meta[key] = value - meta = {} - meta['dataset'] = dataset - meta['station'] = df['station'].values[0] - meta['latitude'] = df['latitude[unit="degrees_north"]'].values[0] - meta['longitude'] = df['longitude[unit="degrees_east"]'].values[0] + meta[variable] = var_meta + df = pd.read_csv(buffer, index_col=0, parse_dates=True) + df = df.replace(float(var_meta["undef"]), np.nan) - # drop the non-data columns - dropcols = ['time', 'station', 'latitude[unit="degrees_north"]', - 'longitude[unit="degrees_east"]'] - df = df.drop(columns=dropcols) + data[variable] = df["Data"] - # column names are like T2M[unit="K"] by default. extract the unit - # for the metadata, then rename col to just T2M - units = {} - rename = {} - for col in df.columns: - name, _ = col.split("[", maxsplit=1) - unit = col.split('"')[1] - units[name] = unit - rename[col] = name + # copy lat/lon to the top level, for consistency + # with other iotools functions + meta["latitude"] = float(var_meta["lat"]) + meta["longitude"] = float(var_meta["lon"]) - meta['units'] = units - df = df.rename(columns=rename) + df = pd.DataFrame(data) + df.index = df.index.tz_localize("UTC") if map_variables: + for col in df.columns: + if col in UNITS: + convert = UNITS[col] + df[col] = convert(df[col]) + df = df.rename(columns=VARIABLE_MAP) return df, meta diff --git a/tests/iotools/test_merra2.py b/tests/iotools/test_merra2.py index 55f28b04e2..a563ad0535 100644 --- a/tests/iotools/test_merra2.py +++ b/tests/iotools/test_merra2.py @@ -6,7 +6,7 @@ import pytest import pvlib import os -import requests +from requests.exceptions import HTTPError from tests.conftest import RERUNS, RERUNS_DELAY, requires_earthdata_credentials @@ -25,11 +25,11 @@ def params(): @pytest.fixture def expected(): - index = pd.date_range("2020-06-01 15:30", "2020-06-01 20:30", freq="h", + index = pd.date_range("2020-06-01 15:30", "2020-06-01 19:30", freq="h", tz="UTC") - index.name = 'time' - albedo = [0.163931, 0.1609407, 0.1601474, 0.1612476, 0.164664, 0.1711341] - ghi = [ 930., 1002.75, 1020.25, 981.25, 886.5, 743.5] + index.name = 'Timestamp (UTC)' + albedo = [0.163931, 0.1609407, 0.1601474, 0.1612476, 0.164664] + ghi = [ 930., 1002.75, 1020.25, 981.25, 886.5] df = pd.DataFrame({'albedo': albedo, 'ghi': ghi}, index=index) return df @@ -38,10 +38,40 @@ def expected(): def expected_meta(): return { 'dataset': 'M2T1NXRAD.5.12.4', - 'station': 'GridPointRequestedAt[40.010N_80.010W]', 'latitude': 40.0, 'longitude': -80.0, - 'units': {'ALBEDO': '1', 'SWGDN': 'W m-2'} + 'ALBEDO': { + 'prod_name': 'M2T1NXRAD.5.12.4', + 'doi': '10.5067/Q9QMY5PBNV1T', + 'param_short_name': 'ALBEDO', + 'param_name': 'Surface albedo, time average', + 'unit': '1', + 'undef': '1e+15', + 'begin_time': '2020-06-01 15:30:00', + 'end_time': '2020-06-01 19:30:00', + 'lat': '40.0', + 'lon': '-80.0', + 'lat_resolution': '0.5', + 'lon_resolution': '0.625', + 'mean': '1.6219e-01', + # 'Request_time': '2026-08-05 12:35:06' + }, + 'SWGDN': { + 'prod_name': 'M2T1NXRAD.5.12.4', + 'doi': '10.5067/Q9QMY5PBNV1T', + 'param_short_name': 'SWGDN', + 'param_name': 'Surface incoming shortwave flux, time average', + 'unit': 'W m-2', + 'undef': '1e+15', + 'begin_time': '2020-06-01 15:30:00', + 'end_time': '2020-06-01 19:30:00', + 'lat': '40.0', + 'lon': '-80.0', + 'lat_resolution': '0.5', + 'lon_resolution': '0.625', + 'mean': '9.6415e+02', + # 'Request_time': '2026-08-05 12:35:09' + } } @@ -51,6 +81,8 @@ def expected_meta(): def test_get_merra2(params, expected, expected_meta): df, meta = pvlib.iotools.get_merra2(**params) pd.testing.assert_frame_equal(df, expected, check_freq=False) + meta["SWGDN"].pop("Request_time") # this changes from run to run, + meta["ALBEDO"].pop("Request_time") # so don't check it assert meta == expected_meta @@ -61,11 +93,15 @@ def test_get_merra2_map_variables(params, expected, expected_meta): df, meta = pvlib.iotools.get_merra2(**params, map_variables=False) expected = expected.rename(columns={'albedo': 'ALBEDO', 'ghi': 'SWGDN'}) pd.testing.assert_frame_equal(df, expected, check_freq=False) + meta["SWGDN"].pop("Request_time") # this changes from run to run, + meta["ALBEDO"].pop("Request_time") # so don't check it assert meta == expected_meta +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_merra2_error(): - with pytest.raises(ValueError, match='must be in the same year'): + with pytest.raises(HTTPError, match='Unauthorized for url'): pvlib.iotools.get_merra2(40, -80, '2019-12-31', '2020-01-02', username='anything', password='anything', dataset='anything', variables=[]) @@ -81,6 +117,8 @@ def test_get_merra2_timezones(params, expected, expected_meta): params[key] = dt.tz_localize('UTC').tz_convert('Etc/GMT+5') df, meta = pvlib.iotools.get_merra2(**params) pd.testing.assert_frame_equal(df, expected, check_freq=False) + meta["SWGDN"].pop("Request_time") # this changes from run to run, + meta["ALBEDO"].pop("Request_time") # so don't check it assert meta == expected_meta @@ -89,7 +127,7 @@ def test_get_merra2_timezones(params, expected, expected_meta): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_merra2_bad_credentials(params, expected, expected_meta): params['username'] = 'nonexistent' - with pytest.raises(requests.exceptions.HTTPError, match='Unauthorized'): + with pytest.raises(HTTPError, match='Unauthorized'): pvlib.iotools.get_merra2(**params) @@ -98,7 +136,7 @@ def test_get_merra2_bad_credentials(params, expected, expected_meta): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_merra2_bad_dataset(params, expected, expected_meta): params['dataset'] = 'nonexistent' - with pytest.raises(requests.exceptions.HTTPError, match='404'): + with pytest.raises(HTTPError, match='Forbidden for url'): pvlib.iotools.get_merra2(**params) @@ -107,5 +145,15 @@ def test_get_merra2_bad_dataset(params, expected, expected_meta): @pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) def test_get_merra2_bad_variables(params, expected, expected_meta): params['variables'] = ['nonexistent'] - with pytest.raises(requests.exceptions.HTTPError, match='400'): + with pytest.raises(HTTPError, match='Forbidden for url'): pvlib.iotools.get_merra2(**params) + + +@requires_earthdata_credentials +@pytest.mark.remote_data +@pytest.mark.flaky(reruns=RERUNS, reruns_delay=RERUNS_DELAY) +def test_get_merra2_multiple_datasets(params): + params['variables'] = ["SWGDN", "T2M"] + params["dataset"] = ["M2T1NXRAD.5.12.4", "M2T1NXSLV.5.12.4"] + df, meta = pvlib.iotools.get_merra2(**params) + assert set(df.columns) == {"ghi", "temp_air"}