Merge pull request #342 from sul-dlss/synonymEarthworksProduction

EarthWorks production:adding synonyms and synonym handling

earthworks-aardvark-prod/schema.xml

@@ -96,6 +96,7 @@
       </analyzer>
       <analyzer type="query">
         <tokenizer class="solr.StandardTokenizerFactory"/>
+        <filter class="solr.SynonymGraphFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords_en.txt"/>
         <filter class="solr.LowerCaseFilterFactory"/>
         <filter class="solr.EnglishPossessiveFilterFactory"/>

earthworks-aardvark-prod/synonyms.txt

@@ -0,0 +1,386 @@
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#-----------------------------------------------------------------------
+#some test synonym mappings unlikely to appear in real input text
+aaa => aaaa
+bbb => bbbb1 bbbb2
+ccc => cccc1,cccc2
+a\=>a => b\=>b
+a\,a => b\,b
+fooaaa,baraaa,bazaaa
+
+# Some synonym groups specific to this example
+GB,gib,gigabyte,gigabytes
+MB,mib,megabyte,megabytes
+Television, Televisions, TV, TVs
+#notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
+#after us won't split it into two words.
+
+# Synonym mappings can be used for spelling correction too
+pixima => pixma
+
+# Copied from OGP LCSH synonyms, with duplicates removed from multi-way expansion synonyms 
+Harbors,Pier,Quay,Warf
+
+Hydrocarbons,CNG,Coal,ethanol,gas,gasoline,kerosene,LNG,oil,OPEC,petrol,Petroleum,propane
+
+Walls,Fences,Fortification,fortresses,Gates,Hedges,Seawall,Wall
+
+Roads,arteries,automobiles,Avenue,Avenues,beltways,Boulevards,driveways,expressway,expressways,Freeways,highway,highways,intersections,interstate,interstates,lanes,motorways,road,Roadblocks,Roadways,route,Routes,Speedlimits,Street,Streets,Thoroughfares,Tollbooths,way
+
+Schools,academies,College,colleges,colleges/universities,Education,preschool,preschools,school,universities,university,University/Colleges
+
+Pedestrians,Curbs,Roads,Footpaths,handicap,Pavements,Pedestrian,sidewalk,Sidewalks,walk,Walking,Walkways
+
+Atmosphere,Air,climate,Climatology,climatologymeteorology,Cloudiness,Clouds,Evapotrans,Evapotranspiration,haze,humidity,hurricane,Isohyets,Meteorological,Meteorology,Precipitation,Rain,rainfall,Sunshine,temperature,weather,wind
+
+Altitudes,Altitude,Aspect,bathymetry,breaklines,contour,Contours,DEM,depths,DTM,echosounder,elevations,GTOPO30,highpoint,hillshaded,Hypsography,ruggedness,SRTM,terrain,topographic,topography,Topoquads
+
+History,ancestry,Historic,historical,History,Memorial,Memorials,Ruins
+
+Industries,chemists,Factories,Buildings,Factory,industrial,industry,Iron-works,Manufacture,refineries
+
+Water-supply,Drinking,Water-pipes,Waterworks
+
+watersheds,catchments,hydrology,drainage,drainages,watershed
+
+fishes,anadromous,fish,fisheries,fishing,oncorhynchus,spawn
+
+census,ageb,cbsa
+
+mountains,hill,hills,mount,mountain,peak,peaks,ridge,ridgelines,geology,summit
+
+bicycles,bicycle,bike
+
+sewerage,drains,manhole,manholes,sewage,sewer,sewer-pipe,sewers
+
+floods,dfirm,finance,flood,flooding,floodplains,watersheds,floodwalls,walls,floodway,nfip,finance,sfha
+
+agriculture,agricultural,appellation,commerce,broilers,calves,cattle,chickens,corn,cotton,cranberies,cranberries,cranberry,food,croplands,crops,farm,farming,farms,fertilizers,grazing,herbicide,livestock,orchards,pesticide,plantations,poultry,rice,soybeans,vegetables,viticulture,wheat,wine,wineries
+
+hydrology,fwhydrography,hydrographic,hydrography,hydrologic,hydrological,runoff,water-holes
+
+buildings,building,buildingsr,castles,chimneys,decks,garage,garages,halls,hotels,mansions,porches,rooftop,stories
+
+buses,bus,ctran,grta
+
+geology,bedrock,erosion,eruptions,eskers,cryosphere,faults,fjords,cryosphere,geomorphology,geoscience,isopach,kettlehole,cryosphere,landslides,lava,lithology,petrology,rocks,volcanoes
+
+demography,births,caste,citizenship,deaths,demographic,demographics,ethnicity,family,females,fertility,foreign-born,household,households,illiteracy,schools,language,languages,literacy,males,minorities,mortality,multilingualism,people,population,populations,unemployment,finance
+
+machinery,mechanical,tractors
+
+wetlands,bog
+
+food,burgers,hamburgers,restaurants,commerce,supermarket
+
+coasts,accretion,geology,beach,beaches,breakwater,capes,coast,coastal,coastline,coastlines,intracoastal,isthmus,isthmuses,nearshore,peninsula,peninsulas,seashore,seashores,shore,shoreline,shorelines,spit
+
+cryosphere,glaciers,geology,ice,icebergs,nunataks,permafrost,snow,atmosphere,snowsheds
+
+groundwater,aquifers,geology,geology,oases,springs
+
+youth,child,children,teen
+
+plants,eelgrass,gardens,agriculture,seagrass,vegetation
+
+earthquakes,geology,magnitude,magnitudes,paleoseismology,seismology
+
+recreation,amphitheaters,amphitheatre,aquariums,arenas,athletics,auditoriums,ballfields,campgrounds,casinos,cinemas,clubs,entertainment,exhibitions,golf,hiking,mountaineering,nightclubs,nightlife,opera,play,playground,playgrounds,racetracks,racing,rail-trails,railroads,regattas,resorts,rinks,speedways,sports,stadium,stadiums,swimming,tennis,theaters,theatres,trail,trails,venues,yachts,ymcas,zoos
+
+photography,landsat,naturalvue,ortho,orthoimagery,ortho-imagery,orthophoto,orthophotography,othophoto
+
+telecommunication,broadcasting,cable,celltowers,telecommunications,telegraph,telephone,telephones,television
+
+railroads,amtrak,fixed-guideway,lirr,metro,metros,monorail,rail,railroad,rails,railway,railways,railyards,street-railroads,subway,subways,train,trains
+
+airports,aircraft,airfield,airfields,airlines,airplanes,airstrips,enplanements,helicopters,icao,runway,runways
+
+pollution,contaminant,contamination,fouling,nox,ozone,atmosphere,pm10,pollutants,reclamation,smokestacks,sox
+
+commerce,billboards,businesses,industries,cargo,commercial,companies,industries,containerization,currencies,customers,employment,freeports,harbors,freight,imports,malls,marketing,markets,occupations,pricing,purchases,rates,retail,revenues,sales,shipping,shops,trade
+
+rivers,brook,hydrology,brooks,creek,creeks,falls,rapids,river,riverbeds,riverine,riverline,rivulet,runs,stream,streambed,streambeds,streamflow,streams,streamways,tributary,wadies,waterfall,waterfalls
+
+archaeology,antiquities,relics
+
+finance,atms,bank,banks,income,insurance,tax,taxation,taxes
+
+crises,disaster
+
+# Copied from OGP ISO synonyms - with duplicates removed
+
+# Invertebrates line changed to be expansion in a single direction
+Invertebrates => Invertebrates,Lobsters,Mosquitoes,Shellfish
+
+Pollution,Contaminant,Contamination,Fouling,Atmosphere,NOx,Atmosphere,Ozone,Atmosphere,PM10,Atmosphere,Pollutants,Reclamation,smokestacks,SOx,Atmosphere
+
+Older people,elder,elderly,retirement,senior
+
+Buses,Bus,CTRAN,GRTA
+
+Photography,landsat,NaturalVue,ortho,Orthoimagery,ortho-imagery,orthophoto,Orthophotography,Othophoto
+
+Mountains,hill,hills,mount,mountain,peak,Peaks,ridge,ridgelines,Geology,summit
+
+Water-supply,Drinking,Water-pipes,Waterworks
+
+Youth,child,children,teen
+
+Coasts,accretion,Geology,beach,Beaches,Breakwater,capes,coast,Coastal,Coastline,coastlines,Intracoastal,isthmus,isthmuses,nearshore,peninsula,peninsulas,Seashore,seashores,shore,shoreline,shorelines,spit
+
+Refuse, refuse disposal,garbage,hazmat,Landfill,Landfills,litter,recycling,trash,waste,wastewater
+
+climatologyMeterorologyAtmosphere,Atmosphere,Air,climate,Climatology,climatologymeteorology,Cloudiness,Clouds,Evapotrans,Evapotranspiration,haze,humidity,hurricane,Isohyets,Meteorological,Meteorology,Precipitation,Rain,rainfall,Sunshine,temperature,weather,wind
+
+Census,AGEB,cbsa,Census
+
+Recreation,Amphitheaters,Amphitheatre,aquariums,Arenas,athletics,Auditoriums,ballfields,Campgrounds,casinos,cinemas,Clubs,entertainment,Exhibitions,golf,Hiking,Mountaineering,"Museums,",Nightclubs,nightlife,opera,play,Playground,Playgrounds,Racetracks,Racing,Rail-trails,Railroads,recreation,Regattas,resorts,rinks,speedways,Sports,Stadium,Stadiums,swimming,tennis,theaters,theatres,trail,Trails,venues,yachts,YMCAs,Zoos
+
+Crises,disaster
+
+Agriculture,agricultural,Appellation,Commerce,Broilers,Calves,Cattle,Chickens,Corn,Cotton,Cranberies,Food,Cranberries,cranberry,Croplands,Crops,farm,farming,Farms,fertilizers,grazing,herbicide,Livestock,Orchards,pesticide,Plantations,Poultry,Rice,Soybeans,Vegetables,Viticulture,Wheat,Wine,wineries
+
+Geology,bedrock,Erosion,eruptions,Eskers,Cryosphere,faults,fjords,Cryosphere,Geomorphology,geoscience,isopach,Kettlehole,Cryosphere,Landslides,lava,Lithology,Petrology,Rocks,Volcanoes
+
+Airports,aircraft,airfield,airfields,airlines,Airplanes,airstrips,enplanements,Helicopters,ICAO,runway,runways
+
+Real property,apartments,Buildings,Assessing,Assessor,cadastral,deeds,Easement,Easements,encumbrance,erven,Housing,Landowners,leasehold,leases,lot,Lots,parcel,Parcels,Property,servitude
+
+"Justice, Administration of",Court,courthouses,Courts,Judicial,Justice,magistrate
+
+Hydrology,FWHYDROGRAPHY,Hydrographic,Hydrography,hydrologic,Hydrological,Runoff,water-holes
+
+Floods,DFIRM,Finance,Flood,Flooding,Floodplains,Watersheds,floodwalls,Walls,Floodway,NFIP,Finance,SFHA
+
+History,ancestry,Historic,historical,Memorial,Memorials,Ruins
+
+Earthquakes,Geology,Magnitude,magnitudes,Paleoseismology,Seismology
+
+utilitiesCommunications,Telecommunication,Broadcasting,cable,celltowers,Telecommunication,Telecommunications,Telegraph,telephone,telephones,television
+
+environment,Environmental protection,Conservation,EPA,NHESP
+
+transportation, Terminals (Transportation),hubs,seaports,terminals
+
+Public safety,assaults,Emergency,FEMA,Fire,Firebreaks,Firehouses,Fires,Firewall,firewalls,Fuelbreaks,Hazard,Hazards,Hydrants,Larceny,shelter,shelters,signals,Signs,Streetlights,Traffic,Wildfire
+
+Bodies of water,Bayous,channel,channels,Freshwater,gulfs,Coasts,Oshanas,pond,Ponds,sea,Seas,surfacewater
+
+Schools,academies,College,colleges,colleges/universities,Education,preschool,preschools,school,universities,university,University/Colleges
+
+Grids (Cartography),Equator,graticule,grid100,grid50,grids,Latitude,Latitude/Longitude,latitudes,Longitude,longitudes,utm
+
+Railroads,Amtrak,fixed-guideway,LIRR,metro,metros,monorail,rail,railroad,Rails,railway,railways,Railyards,Street-railroads,subway,Subways,train,Trains
+
+Pedestrians,Curbs,Roads,Footpaths,handicap,Pavements,Pedestrian,sidewalk,Sidewalks,walk,Walking,Walkways
+
+structure,Hydraulic structures,aqueduct,aqueducts,Barrages,Breakwaters,Coasts,canal,Canals,Culverts,Dam,Dams,Embankments,Flumes,Gryone,Levees,Penstocks,Qanats,sluices,Weir,Weirs
+
+Groundwater,Aquifers,Geology,Oases,Springs
+
+# This line originally had "buildings" which seems an odd term to make substitutions for, so it was removed here
+Religious institutions,abbeys,Buddhism,chapels,church,churches,Convents,monasteries,Mosques,religion,religious,shrines,synagogues,Temples,Worship
+
+Commerce,Billboards,businesses,Industries,Cargo,commercial,companies,Containerization,currencies,customers,employment,freeports,Harbors,freight,imports,malls,marketing,Markets,Occupations,pricing,purchases,rates,retail,revenues,sales,Shipping,shops,trade
+
+Land use,Cemeteries,cemetery,graveyards,Greenways,grounds,landscaping,Landuse,pave,paved,Pavement,Planimetric,Planning,Plazas,PLSS,rangeland,Redevelopment,Surveying,unpaved,Zoning
+
+Archaeology,Antiquities,Relics
+
+Watersheds,Catchments,Hydrology,Drainage,drainages,Watershed,Watersheds
+
+Protected areas,parkland,Parklands,Parks,Parkways,preserves
+
+Demography,births,Caste,Citizenship,deaths,demographic,demographics,Ethnicity,Family,females,fertility,Foreign-Born,household,households,illiteracy,Schools,language,Languages,Literacy,males,Minorities,mortality,Multilingualism,people,Population,populations,Unemployment,Finance
+
+Public institutions,Jails,Libraries,Buildings,Library,museums,penitentiaries,Prison,Prisons
+
+biota, Ecology & environment,biome,ecological,Ecology,ecoregions,Ecosystem,ecosystems,environmental,evolutionary,habitat,Habitats,reef,Reefs,sustainability,Wildlife,Zoology
+
+Marine mammals,Cetaceans,Dolphins,Porpoises,Whales
+
+Metropolitan areas,metropolitan,urban,Urbanization
+
+Forests,forestry,arboretums,Plants,Evergreen,oak,Pine,Timberline,tree,Trees
+
+Roads,arteries,automobiles,Avenue,Avenues,beltways,Boulevards,driveways,expressway,expressways,Freeways,highway,highways,intersections,interstate,interstates,lanes,motorways,road,Roadblocks,Roads,Roadways,route,Routes,Speedlimits,Street,Streets,Thoroughfares,Tollbooths,way
+
+health,Public health,Clinics,dentists,doctor,doctors,ER,Gonorrhea,HIV,hospital,hospitals,immunization,MDPH,medical,Medicare,medicine,pharmacies,Physicians,polyclinics,quarantine,Sanitation,Syphilis,veterinarians
+
+elevation,Altitudes,Altitude,Aspect,bathymetry,breaklines,contour,Contours,DEM,depths,DTM,echosounder,elevations,GTOPO30,highpoint,hillshaded,Hypsography,ruggedness,SRTM,terrain,topographic,topography,Topoquads
+
+Capitals (Cities),capitals
+
+Wetlands,Bog
+
+"Names, Geographical",gazetteer,Gazetteers,GNS,label,Labels,Lable
+
+Walls,Fences,Fortification,fortresses,Buildings,Gates,Hedges,Seawall,Wall
+
+Finance,atms,bank,banks,income,Insurance,Tax,Taxation,Taxes
+
+Boats,boating,Boat,Boating,Ferries,ferry,lighthouse,Coasts,lighthouses,Coasts,locks,Marinas,Shipwrecks,waterway,Waterways
+
+intelligenceMilitary,International relations,Armistices,Crises,Army,bioterrorism,Crises,CIA,Installation,Installations,Military,Navy
+
+boundaries,Administrative,political divisions,administrative,administrativeBoundaries,barrios,border,Borders,boundariesAdministrative,canton,cantons,colony,Counties,countries,county,Countyline,distirct,district,Districts,emirates,lgas,Macroregions,nations,precinct,precincts,Prefectures,province,Provinces,Redistrict,sovereignties,subdistricts,Subdivision,subdivisions,submunicipalities,subregions
+
+Politics,government,commonwealth,Congress,Congressional,constituency,councils,deregulation,election,Elections,Elections--Massachusetts,Electoral,embassies,Governates,Government,legislation,Legislative,Legislators,Legislature,Parliament,political,Regulation,republic,republics,Senate,States,vote,Voting
+
+Machinery,mechanical,Tractors
+
+Cities,towns,borough,Boroughs,hamlets,Municipalities,municipality,municipios,suburbs,Town,Townlands,towns/ville,Villages,ville
+
+Hydrocarbons,CNG,Coal,ethanol,gas,gasoline,kerosene,LNG,oil,OPEC,petrol,Petroleum,propane
+
+Indigenous peoples,tribes
+
+Land cover,brush,Plants,Caves,Cliffs,continents,coves,Desert,Atmosphere,Deserts,earthCover,Fields,Gorges,Grassland,Plants,Grasslands,Plants,Heathlands,Plants,island,Islands,Landcover,Landform,Landforms,Landscape,Landscapes,meadow,palisade,plain,plains,Prairies,tundra,Tundras,Wilderness
+
+Harbors,Pier,Quay,Warf
+
+Automobile parking,Parking
+
+Plants,eelgrass,gardens,Agriculture,seagrass,Vegetation
+
+Buildings,building,Buildingsr,castles,Chimneys,Decks,garage,garages,halls,Hotels,mansions,office,Porches,rooftop,stories
+
+Industries,chemists,Factories,Buildings,Factory,Buildings,industrial,industry,Iron-works,Manufacture,refineries
+
+Mines,mineral resources,Mineral,Geology,minerals,mining,prospects,quarries,Quarry,USBM
+
+inlandWaters,Rivers,brook,Hydrology,brooks,Hydrology,creek,Hydrology,Creeks,Hydrology,falls,Rapids,river,Hydrology,river/fleuve,Hydrology,riverbeds,Riverine,Riverline,rivers,Hydrology,rivulet,Hydrology,runs,stream,Hydrology,streambed,Streambeds,Streamflow,streams,Hydrology,streamways,tributary,Hydrology,Wadies,Hydrology,Waterfall,Waterfalls
+
+Food,burgers,hamburgers,restaurants,Commerce,supermarket,Commerce
+
+Cryosphere,Glaciers,Geology,Ice,Icebergs,Nunataks,Geology,permafrost,Snow,Atmosphere,Snowsheds
+
+Bicycles,Bicycle,Bike
+
+Postal service,address,Roads,addresses,Roads,CEP8,mail,PLZgrenzen,postal,postcodes,zip,zip+4,zipcode,zipcodes
+
+Human settlements,condominiums,Dormitories,Dwellings,homelessness,homes,Huts,inhabitancy,Neighborhood,Neighborhoods,residences,Settlement,settlements,squatters
+
+Sewerage,drains,manhole,Manholes,sewage,Sewer,Sewer-pipe,Sewers
+
+Fishes,Anadromous,Fish,Fisheries,Fishing,Oncorhynchus,Spawn,Wildlife/Fisheries
+
+Territories, possessions,Annexation,Annexations,territory
+
+Electric power systems,biodiesel,Electric,electricity,energy,fuel,Fuels,Generators,hydrogen,megawatt-hours,MWh,Power-plants,Substation,substations,Windmills
+
+# Copied from OGP placename synonyms
+Alabama,AL,Ala 
+
+Alaska,AK,Alaska,Alas
+
+Arizona,AZ,Ariz
+
+Arkansas,AR,Ark 
+
+California,CA,Calif 
+
+Colorado,CO,Colo 
+
+Connecticut,CT,Conn 
+
+Delaware,DE,Del 
+
+District of Columbia,DC 
+
+Florida,FL,Fla 
+
+Georgia,GA
+
+Hawaii,HI 
+
+Idaho,ID 
+
+Illinois,IL,Ill 
+
+Indiana,IN,Ind 
+
+Iowa,IA,Iowa 
+
+Kansas,KS,Kans,Kan 
+
+Kentucky,KY 
+
+Louisiana,LA 
+
+Maine,ME,Maine 
+
+Maryland,MD
+
+Massachusetts,MA,Mass 
+
+Michigan,MI,Mich 
+
+Minnesota,MN,Minn 
+
+Mississippi,MS,Miss 
+
+Missouri,MO 
+
+Montana,MT,Mont 
+
+Nebraska,NE,Nebr,Neb 
+
+Nevada,NV,Nev 
+
+New Hampshire,NH 
+
+New Jersey,NJ 
+
+New Mexico,NM,N Mex 
+
+New York,NY 
+
+North Carolina,NC,N Car
+
+North Dakota,ND,N Dak 
+
+Ohio,OH 
+
+Oklahoma,OK,Okla 
+
+Oregon,OR,Oreg,Ore 
+
+Pennsylvania,PA,Penn
+
+Rhode Island,RI
+
+South Carolina,SC,S Car
+
+South Dakota,SD,S Dak
+
+Tennessee,TN,Tenn
+
+Texas,TX,Tex,Texas 
+
+Utah,UT,Utah
+
+Vermont,VT,Vt
+
+Virginia,VA,Virg 
+
+Washington,WA,Wash
+
+West Virginia,WV,W Va,WVa 
+
+Wisconsin,WI,Wis,Wisc
+
+Wyoming,WY,Wyo
+
+Puerto Rico,PR