diff --git a/rmf_building_map_tools/pit_crew/pit_crew.py b/rmf_building_map_tools/pit_crew/pit_crew.py index 459e81a95..8e47d5a5d 100644 --- a/rmf_building_map_tools/pit_crew/pit_crew.py +++ b/rmf_building_map_tools/pit_crew/pit_crew.py @@ -72,7 +72,7 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) -ModelNames = namedtuple("ModelNames", ["model_name", "author_name"]) +ModelNames = namedtuple("ModelNames", ["model_name", "author_name", "upload_date"]) def remove_spaces(original): @@ -393,7 +393,7 @@ def get_author_to_model_dict(model_name_tuples, lower=True): """ output = {} - for (model_name, author_name) in model_name_tuples: + for (model_name, author_name, _) in model_name_tuples: if lower: model_name = model_name.lower() author_name = author_name.lower() @@ -422,7 +422,7 @@ def get_model_to_author_dict(model_name_tuples, lower=True): """ output = {} - for (model_name, author_name) in model_name_tuples: + for (model_name, author_name, _) in model_name_tuples: if lower: model_name = model_name.lower() author_name = author_name.lower() @@ -713,9 +713,11 @@ def download_model_fuel_tools(model_name, author_name, (model_name, export_path)) return True - except Exception as e: - logger.error("Could not download model '%s'! %s" % (model_name, e)) + except KeyError as e: return False +# except Exception as e: +# logger.error("Could not download model '%s'! %s" % (model_name, e)) +# return False def _construct_license(fuel_metadata_dict): @@ -844,7 +846,7 @@ def build_and_update_cache(cache_file_path=None, write_to_cache=True, while status == 200 and not break_flag: logger.info("Fetching page: %d" % page) - resp = requests.get("%s?page=%d" % (url_base, page)) + resp = requests.get("%s?page=%d&per_page=100" % (url_base, page)) status = resp.status_code page += 1 @@ -852,17 +854,18 @@ def build_and_update_cache(cache_file_path=None, write_to_cache=True, for model in json.loads(resp.text): model_name = model.get("name", "") author_name = model.get("owner", "") + upload_date = model.get("upload_date", "") # If a cached model was found, halt - if (model_name, author_name) in old_cache['model_cache']: - logger.info("Cached model found! " + if (model_name, author_name, upload_date) in old_cache['model_cache']: + logger.info("Cached model found! ", (model_name, author_name, upload_date), "Halting Fuel traversal...") break_flag = True break # Otherwise, add it to the cache else: new_cache_count += 1 - old_cache['model_cache'].add((model_name, author_name)) + old_cache['model_cache'].add((model_name, author_name, upload_date)) old_cache['fuel_cache'].append(model) else: break @@ -986,4 +989,4 @@ def sync_sdf(model_name, extract_path): except Exception as e: logger.error("Syncing of names for %s failed! %s" - % (model_name, e)) + % (model_name, e)) \ No newline at end of file