Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Uniformize silent kwargs #224

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pynsee/download/download_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@


@save_df(day_lapse_max=90)
def download_file(id, variables=None, update=False, silent=False):
def download_file(id, variables=None, update=False, silent=True):
"""User level function to download files from insee.fr

Args:
Expand Down
17 changes: 13 additions & 4 deletions pynsee/geodata/_get_geodata.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,24 @@
from pynsee.geodata._geojson_parser import _geojson_parser

from pynsee.utils.save_df import save_df
from pynsee.utils.requests_params import _get_requests_headers, _get_requests_proxies
from pynsee.utils.requests_params import (
_get_requests_headers,
_get_requests_proxies,
)

import logging

logger = logging.getLogger(__name__)


@save_df(day_lapse_max=90)
def _get_geodata(
id, polygon=None, update=False, silent=False, crs="EPSG:3857", crsPolygon="EPSG:4326"
id,
polygon=None,
update=False,
silent=True,
crs="EPSG:3857",
crsPolygon="EPSG:4326",
):
"""Get geographical data with identifier and from IGN API

Expand Down Expand Up @@ -60,7 +69,7 @@ def _get_geodata(
Version = "2.0.0"

# make the query link for ign
#geoportail = "https://wxs.ign.fr/{}/geoportail".format(topic)
# geoportail = "https://wxs.ign.fr/{}/geoportail".format(topic)
geoportail = f"https://data.geopf.fr/{service.lower()}/ows?"
Service = "SERVICE=" + service + "&"
version = "VERSION=" + Version + "&"
Expand All @@ -70,7 +79,7 @@ def _get_geodata(

link0 = (
geoportail
#+ "/wfs?"
# + "/wfs?"
+ Service
+ version
+ request
Expand Down
7 changes: 5 additions & 2 deletions pynsee/geodata/get_geodata_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@

from pynsee.utils.save_df import save_df


@save_df(day_lapse_max=90)
def get_geodata_list(update=False, silent=False):
def get_geodata_list(update=False, silent=True):
"""Get a list of geographical limits of French administrative areas from IGN API

Args:
Expand Down Expand Up @@ -44,7 +45,9 @@ def get_geodata_list(update=False, silent=False):
list_first_col = [
col for col in data_full_list.columns if col in list_var
]
list_other_col = [col for col in data_full_list.columns if col not in list_first_col]
list_other_col = [
col for col in data_full_list.columns if col not in list_first_col
]

data_list = data_full_list[list_first_col + list_other_col]
data_list = data_list.drop_duplicates().reset_index(drop=True)
Expand Down
3 changes: 2 additions & 1 deletion pynsee/localdata/get_area_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@
from pynsee.utils._paste import _paste
from pynsee.utils.save_df import save_df


@save_df(day_lapse_max=90)
def get_area_list(area=None, date=None, update=False, silent=False):
def get_area_list(area=None, date=None, update=False, silent=True):
"""Get an exhaustive list of administrative areas : communes, departments, and urban, employment or functional areas

Args:
Expand Down
10 changes: 9 additions & 1 deletion pynsee/localdata/get_area_projection.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import logging

logger = logging.getLogger(__name__)


@lru_cache(maxsize=None)
def _warning_get_area_projection():
logger.info(
Expand All @@ -20,7 +22,11 @@ def _warning_get_area_projection():

@save_df(day_lapse_max=90)
def get_area_projection(
area: str, code: str, date: str, dateProjection: str = None
area: str,
code: str,
date: str,
dateProjection: str = None,
silent: bool = True,
):
"""
Get data about the area (valid at given `date` datetime) projected
Expand All @@ -44,6 +50,8 @@ def get_area_projection(
is supposed to be the current date (ie projection into today's
value)

silent (bool, optional): Set to True, to disable messages printed in log info

Examples:
>>> from pynsee.localdata import get_area_projection
>>> df = get_area_projection(
Expand Down
15 changes: 7 additions & 8 deletions pynsee/localdata/get_ascending_area.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,15 @@

logger = logging.getLogger(__name__)


@save_df(day_lapse_max=90)
def get_ascending_area(
area: str,
code: str,
date: str = None,
type: str = None,
update: bool = False,
silent: bool = False
silent: bool = True,
):
"""
Get information about areas containing a given area
Expand All @@ -39,7 +40,7 @@ def get_ascending_area(
update (bool): locally saved data is used by default. Trigger an update with update=True.

silent (bool, optional): Set to True, to disable messages printed in log info

Examples:
>>> from pynsee.localdata import get_ascending_area
>>> df = get_ascending_area("commune", code='59350', date='2018-01-01')
Expand All @@ -62,7 +63,7 @@ def get_ascending_area(

params_hash = ["get_ascending_area", area, code, date, type]
params_hash = [x if x else "_" for x in params_hash]

INSEE_localdata_api_link = "https://api.insee.fr/metadonnees/V1/geo/"

api_link = INSEE_localdata_api_link + area + f"/{code}/ascendants?"
Expand All @@ -75,10 +76,8 @@ def get_ascending_area(

api_link = api_link + "&".join(params)

request = _request_insee(
api_url=api_link, file_format="application/json"
)

request = _request_insee(api_url=api_link, file_format="application/json")

try:
data = request.json()

Expand All @@ -92,6 +91,6 @@ def get_ascending_area(

except Exception:
logger.error("No data found !")
data_final = pd.DataFrame()
data_final = pd.DataFrame()

return data_final
13 changes: 8 additions & 5 deletions pynsee/localdata/get_descending_area.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,18 @@
from pynsee.utils.save_df import save_df

import logging

logger = logging.getLogger(__name__)


@save_df(day_lapse_max=90)
def get_descending_area(
area: str,
code: str,
date: str = None,
type: str = None,
update: bool = False,
silent: bool = True,
):
"""
Get information about areas contained in a given area
Expand All @@ -36,6 +39,8 @@ def get_descending_area(

update (bool): locally saved data is used by default. Trigger an update with update=True.

silent (bool, optional): Set to True, to disable messages printed in log info

Examples:
>>> from pynsee.localdata import get_area_descending
>>> df = get_descending_area("commune", code='59350', date='2018-01-01')
Expand All @@ -59,7 +64,7 @@ def get_descending_area(

params_hash = ["get_descending_area", area, code, date, type]
params_hash = [x if x else "_" for x in params_hash]

INSEE_localdata_api_link = "https://api.insee.fr/metadonnees/V1/geo/"

api_link = INSEE_localdata_api_link + area + f"/{code}/descendants?"
Expand All @@ -72,9 +77,7 @@ def get_descending_area(

api_link = api_link + "&".join(params)

request = _request_insee(
api_url=api_link, file_format="application/json"
)
request = _request_insee(api_url=api_link, file_format="application/json")

try:
data = request.json()
Expand All @@ -89,6 +92,6 @@ def get_descending_area(

except Exception:
logger.error("No data found !")
data_final = pd.DataFrame()
data_final = pd.DataFrame()

return data_final
11 changes: 4 additions & 7 deletions pynsee/localdata/get_geo_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@

from pynsee.utils.save_df import save_df


@save_df(day_lapse_max=90)
def get_geo_list(geo=None, date=None, update=False, silent=False):
def get_geo_list(geo=None, date=None, update=False, silent=True):
"""Get a list of French geographic areas (communes, departements, regions ...)

Args:
Expand Down Expand Up @@ -179,16 +180,12 @@ def get_geo_list(geo=None, date=None, update=False, silent=False):

for i in range(len(data_all.index)):
if pd.isna(data_all.loc[i, "TITLE_DEP1"]):
data_all.loc[i, "CODE_DEP"] = data_all.loc[
i, "code_dep2"
]
data_all.loc[i, "CODE_DEP"] = data_all.loc[i, "code_dep2"]
data_all.loc[i, "TITLE_DEP"] = data_all.loc[
i, "TITLE_DEP2"
]
else:
data_all.loc[i, "CODE_DEP"] = data_all.loc[
i, "code_dep1"
]
data_all.loc[i, "CODE_DEP"] = data_all.loc[i, "code_dep1"]
data_all.loc[i, "TITLE_DEP"] = data_all.loc[
i, "TITLE_DEP1"
]
Expand Down
52 changes: 33 additions & 19 deletions pynsee/localdata/get_local_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,18 @@
import sys
import datetime

from pynsee.localdata._find_latest_local_dataset import _find_latest_local_dataset
from pynsee.localdata._find_latest_local_dataset import (
_find_latest_local_dataset,
)
from pynsee.localdata._get_insee_local_onegeo import _get_insee_local_onegeo
from pynsee.localdata.get_geo_list import get_geo_list
from pynsee.utils.save_df import save_df

import logging

logger = logging.getLogger(__name__)


@lru_cache(maxsize=None)
def _warning_nivgeo(nivgeo):
if nivgeo == "DEP":
Expand All @@ -29,9 +33,15 @@ def _warning_nivgeo(nivgeo):
elif nivgeo == "FE":
logger.info("By default, the query is on all France territory")


@save_df(day_lapse_max=90)
def get_local_data(
variables, dataset_version, nivgeo="FE", geocodes=["1"], update=False, silent=False
variables,
dataset_version,
nivgeo="FE",
geocodes=["1"],
update=False,
silent=True,
):
"""Get INSEE local numeric data

Expand All @@ -47,7 +57,7 @@ def get_local_data(
update (bool): data is saved locally, set update=True to trigger an update

silent (bool, optional): Set to True, to disable messages printed in log info

Raises:
ValueError: Error if geocodes is not a list

Expand All @@ -74,7 +84,7 @@ def get_local_data(

if isinstance(geocodes, pd.core.series.Series):
geocodes = geocodes.to_list()

if type(geocodes) == str:
geocodes = [geocodes]

Expand All @@ -94,37 +104,41 @@ def get_local_data(
_warning_nivgeo(_warning_nivgeo)
elif nivgeo != "METRODOM":
logger.warning("Please provide a list with geocodes argument !")

#
# LATEST AVAILABLE DATASET OPTION
#
pattern = re.compile('.*latest$')

pattern = re.compile(".*latest$")

if pattern.match(dataset_version):

dataset_version = _find_latest_local_dataset(dataset_version, variables, nivgeo, geocodes[0], update)

list_data_all = []


dataset_version = _find_latest_local_dataset(
dataset_version, variables, nivgeo, geocodes[0], update
)

list_data_all = []

for cdg in trange(len(geocodes), desc="Getting data"):

codegeo = geocodes[cdg]
df_default = pd.DataFrame({"CODEGEO": codegeo, "OBS_VALUE": np.nan}, index=[0])

df_default = pd.DataFrame(
{"CODEGEO": codegeo, "OBS_VALUE": np.nan}, index=[0]
)

try:
df = _get_insee_local_onegeo(
variables, dataset_version, nivgeo, codegeo
)

except Exception as e:
df = df_default

list_data_all.append(df)

data_final = pd.concat(list_data_all).reset_index(drop=True)

if data_final.equals(df_default):
logger.error("Error or no data found !")
logger.error("Error or no data found !")

return data_final
5 changes: 4 additions & 1 deletion pynsee/macrodata/_get_dataset_list_internal.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@


@save_df(day_lapse_max=90)
def _get_dataset_list_internal():
def _get_dataset_list_internal(silent=True):
"""
silent (bool, optional): Set to True, to disable messages printed in log info
"""

zip_file = pkg_resources.resource_stream(
__name__, "data/dataset_list_internal.zip"
Expand Down
Loading
Loading