diff --git a/cookbook/helper/open_data_importer.py b/cookbook/helper/open_data_importer.py index 833ec34e83..92f3f4da16 100644 --- a/cookbook/helper/open_data_importer.py +++ b/cookbook/helper/open_data_importer.py @@ -125,20 +125,38 @@ def import_property(self): def import_supermarket(self): datatype = 'store' + existing_data = {} + for obj in Supermarket.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'): + existing_data[obj['open_data_slug']] = obj + + update_list = [] + create_list = [] + self._update_slug_cache(SupermarketCategory, 'category') - insert_list = [] for k in list(self.data[datatype].keys()): - insert_list.append(Supermarket( + obj = Supermarket( name=self.data[datatype][k]['name'], open_data_slug=k, space=self.request.space - )) + ) + if obj.open_data_slug in existing_data: + obj.pk = existing_data[obj.open_data_slug]['pk'] + update_list.append(obj) + else: + create_list.append(obj) + + total_count = 0 + if self.update_existing and len(update_list) > 0: + Supermarket.objects.bulk_update(update_list, ('name', 'open_data_slug')) + total_count += len(update_list) + + if len(create_list) > 0: + Supermarket.objects.bulk_create(create_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) + total_count += len(create_list) # always add open data slug if matching supermarket is found, otherwise relation might fail - supermarkets = Supermarket.objects.bulk_create(insert_list, unique_fields=('space', 'name',), update_conflicts=True, update_fields=('open_data_slug',)) self._update_slug_cache(Supermarket, 'store') - insert_list = [] for k in list(self.data[datatype].keys()): relations = [] order = 0 @@ -154,68 +172,49 @@ def import_supermarket(self): SupermarketCategoryRelation.objects.bulk_create(relations, ignore_conflicts=True, unique_fields=('supermarket', 'category',)) - return supermarkets + return total_count def import_food(self): - identifier_list = [] datatype = 'food' - for k in list(self.data[datatype].keys()): - identifier_list.append(self.data[datatype][k]['name']) - identifier_list.append(self.data[datatype][k]['plural_name']) - - existing_objects_flat = [] - existing_objects = {} - for f in Food.objects.filter(space=self.request.space).filter(name__in=identifier_list).values_list('id', 'name', 'plural_name'): - existing_objects_flat.append(f[1]) - existing_objects_flat.append(f[2]) - existing_objects[f[1]] = f - existing_objects[f[2]] = f self._update_slug_cache(Unit, 'unit') self._update_slug_cache(PropertyType, 'property') + self._update_slug_cache(SupermarketCategory, 'category') + + existing_data = {} + for obj in Food.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'name', 'open_data_slug'): + existing_data[obj['open_data_slug']] = obj - insert_list = [] - insert_list_flat = [] update_list = [] - update_field_list = [] + create_list = [] + for k in list(self.data[datatype].keys()): - if not (self.data[datatype][k]['name'] in existing_objects_flat or self.data[datatype][k]['plural_name'] in existing_objects_flat): - if not (self.data[datatype][k]['name'] in insert_list_flat or self.data[datatype][k]['plural_name'] in insert_list_flat): - insert_list.append({'data': { - 'name': self.data[datatype][k]['name'], - 'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, - 'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']], - 'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None, - 'open_data_slug': k, - 'space': self.request.space.id, - }}) - # build a fake second flat array to prevent duplicate foods from being inserted. - # trying to insert a duplicate would throw a db error :( - insert_list_flat.append(self.data[datatype][k]['name']) - insert_list_flat.append(self.data[datatype][k]['plural_name']) + + obj = { + 'name': self.data[datatype][k]['name'], + 'plural_name': self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, + 'supermarket_category_id': self.slug_id_cache['category'][self.data[datatype][k]['store_category']], + 'fdc_id': re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None, + 'open_data_slug': k, + 'space': self.request.space.id, + } + + if self.update_existing: + obj['space'] = self.request.space + obj['pk'] = existing_data[obj['open_data_slug']]['pk'] + obj = Food(**obj) + update_list.append(obj) else: - if self.data[datatype][k]['name'] in existing_objects: - existing_food_id = existing_objects[self.data[datatype][k]['name']][0] - else: - existing_food_id = existing_objects[self.data[datatype][k]['plural_name']][0] - - if self.update_existing: - update_field_list = ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ] - update_list.append(Food( - id=existing_food_id, - name=self.data[datatype][k]['name'], - plural_name=self.data[datatype][k]['plural_name'] if self.data[datatype][k]['plural_name'] != '' else None, - supermarket_category_id=self.slug_id_cache['category'][self.data[datatype][k]['store_category']], - fdc_id=re.sub(r'\D', '', self.data[datatype][k]['fdc_id']) if self.data[datatype][k]['fdc_id'] != '' else None, - open_data_slug=k, - )) - else: - update_field_list = ['open_data_slug', ] - update_list.append(Food(id=existing_food_id, open_data_slug=k, )) + create_list.append({'data': obj}) - Food.load_bulk(insert_list, None) - if len(update_list) > 0: - Food.objects.bulk_update(update_list, update_field_list) + total_count = 0 + if self.update_existing and len(update_list) > 0: + Food.objects.bulk_update(update_list, ['name', 'plural_name', 'preferred_unit_id', 'preferred_shopping_unit_id', 'supermarket_category_id', 'fdc_id', 'open_data_slug', ]) + total_count += len(update_list) + + if len(create_list) > 0: + Food.load_bulk(create_list, None) + total_count += len(create_list) self._update_slug_cache(Food, 'food') @@ -243,16 +242,25 @@ def import_food(self): FoodProperty.objects.bulk_create(property_food_relation_list, ignore_conflicts=True, unique_fields=('food_id', 'property_id',)) - return insert_list + update_list + return total_count def import_conversion(self): datatype = 'conversion' - insert_list = [] + self._update_slug_cache(Food, 'food') + self._update_slug_cache(Unit, 'unit') + + existing_data = {} + for obj in UnitConversion.objects.filter(space=self.request.space, open_data_slug__isnull=False).values('pk', 'open_data_slug'): + existing_data[obj['open_data_slug']] = obj + + update_list = [] + create_list = [] + for k in list(self.data[datatype].keys()): # try catch here because sometimes key "k" is not set for he food cache try: - insert_list.append(UnitConversion( + obj = UnitConversion( base_amount=self.data[datatype][k]['base_amount'], base_unit_id=self.slug_id_cache['unit'][self.data[datatype][k]['base_unit']], converted_amount=self.data[datatype][k]['converted_amount'], @@ -261,8 +269,23 @@ def import_conversion(self): open_data_slug=k, space=self.request.space, created_by=self.request.user, - )) + ) + + if obj.open_data_slug in existing_data: + obj.pk = existing_data[obj.open_data_slug]['pk'] + update_list.append(obj) + else: + create_list.append(obj) except KeyError: print(str(k) + ' is not in self.slug_id_cache["food"]') - return UnitConversion.objects.bulk_create(insert_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) + total_count = 0 + if self.update_existing and len(update_list) > 0: + UnitConversion.objects.bulk_update(update_list, ('base_unit', 'base_amount', 'converted_unit', 'converted_amount', 'food',)) + total_count += len(update_list) + + if len(create_list) > 0: + UnitConversion.objects.bulk_create(create_list, ignore_conflicts=True, unique_fields=('space', 'base_unit', 'converted_unit', 'food', 'open_data_slug')) + total_count += len(create_list) + + return total_count diff --git a/cookbook/views/api.py b/cookbook/views/api.py index b16d3045a6..f2aa20cdeb 100644 --- a/cookbook/views/api.py +++ b/cookbook/views/api.py @@ -1621,14 +1621,13 @@ def post(self, request, *args, **kwargs): if selected_datatypes['category']['selected']: response_obj['category'] = data_importer.import_category() if selected_datatypes['property']['selected']: - print('importin properties') response_obj['property'] = data_importer.import_property() if selected_datatypes['store']['selected']: - response_obj['store'] = len(data_importer.import_supermarket()) + response_obj['store'] = data_importer.import_supermarket() if selected_datatypes['food']['selected']: - response_obj['food'] = len(data_importer.import_food()) + response_obj['food'] = data_importer.import_food() if selected_datatypes['conversion']['selected']: - response_obj['conversion'] = len(data_importer.import_conversion()) + response_obj['conversion'] = data_importer.import_conversion() return Response(response_obj)