diff --git a/.env b/.env new file mode 100644 index 00000000..50bfa23d --- /dev/null +++ b/.env @@ -0,0 +1,9 @@ +NEO4J_DIST='enterprise' +NEO4J_VERSION='4.2.0' +APOC_VERSION='4.2.0.0' +DATASTORE_VERSION='4_0' +NEO4J_USER=neo4j +NEO4J_PASSWORD=letmein +BOLT_PORT=7687 +HTTP_PORT=3000 +NEO4J_URI="bolt://localhost:{$BOLT_PORT}" \ No newline at end of file diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..50bfa23d --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +NEO4J_DIST='enterprise' +NEO4J_VERSION='4.2.0' +APOC_VERSION='4.2.0.0' +DATASTORE_VERSION='4_0' +NEO4J_USER=neo4j +NEO4J_PASSWORD=letmein +BOLT_PORT=7687 +HTTP_PORT=3000 +NEO4J_URI="bolt://localhost:{$BOLT_PORT}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 70cf5c32..7b9793ce 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,7 @@ neo4j-version test/tck/* !test/tck/.gitkeep -.history \ No newline at end of file +.history + +.download_cache +neo4j \ No newline at end of file diff --git a/.vscode/perl-lang/home/daniel/Workspaces/Javascript/neo4j-graphql-js/scripts/helpers/regex.pl b/.vscode/perl-lang/home/daniel/Workspaces/Javascript/neo4j-graphql-js/scripts/helpers/regex.pl new file mode 100644 index 00000000..796122bb --- /dev/null +++ b/.vscode/perl-lang/home/daniel/Workspaces/Javascript/neo4j-graphql-js/scripts/helpers/regex.pl @@ -0,0 +1 @@ +{"vars":[{"line":0,"kind":2,"name":"strict","containerName":""},{"line":1,"name":"$regex","containerName":null,"kind":13,"localvar":"my","defintion":"my"},{"line":4,"name":"@ARGV","containerName":null,"kind":13}],"version":3} \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..65f4c1fc --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "avaExplorer.cwd": "test" +} diff --git a/example/apollo-federation/gateway.js b/example/apollo-federation/gateway.js index 6e4e0226..b9fe903e 100644 --- a/example/apollo-federation/gateway.js +++ b/example/apollo-federation/gateway.js @@ -5,16 +5,16 @@ import { inventorySchema } from './services/inventory'; import { productsSchema } from './services/products'; import { reviewsSchema } from './services/reviews'; import neo4j from 'neo4j-driver'; +import dotenv from 'dotenv'; + +dotenv.config(); // The schema and seed data are based on the Apollo Federation demo // See: https://github.com/apollographql/federation-demo const driver = neo4j.driver( process.env.NEO4J_URI || 'bolt://localhost:7687', - neo4j.auth.basic( - process.env.NEO4J_USER || 'neo4j', - process.env.NEO4J_PASSWORD || 'letmein' - ) + neo4j.auth.basic(process.env.NEO4J_USER, process.env.NEO4J_PASSWORD) ); // Start Accounts diff --git a/example/apollo-server/authScopes.js b/example/apollo-server/authScopes.js index 4f1d244b..ccd83dff 100644 --- a/example/apollo-server/authScopes.js +++ b/example/apollo-server/authScopes.js @@ -1,3 +1,4 @@ +import { gql } from 'apollo-server'; import { makeAugmentedSchema } from '../../src/index'; import { ApolloServer } from 'apollo-server'; import neo4j from 'neo4j-driver'; @@ -11,15 +12,15 @@ import neo4j from 'neo4j-driver'; // JWT_SECRET // oqldBPU1yMXcrTwcha1a9PGi9RHlPVzQ -const typeDefs = ` -type User { +const typeDefs = gql` + type User { userId: ID! name: String -} + } -type Business { + type Business { name: String -} + } `; const schema = makeAugmentedSchema({ diff --git a/example/apollo-server/bookmarks.js b/example/apollo-server/bookmarks.js index 5dbba301..326e9c95 100644 --- a/example/apollo-server/bookmarks.js +++ b/example/apollo-server/bookmarks.js @@ -1,47 +1,47 @@ +import { gql } from 'apollo-server'; import { makeAugmentedSchema } from '../../src/index'; import { ApolloServer } from 'apollo-server'; import neo4j from 'neo4j-driver'; -const typeDefs = ` -type Person { - _id: Long! - born: Int - name: String! - acted_in: [Movie] @relation(name: "ACTED_IN", direction: "OUT") - ACTED_IN_rel: [ACTED_IN] - directed: [Movie] @relation(name: "DIRECTED", direction: "OUT") - produced: [Movie] @relation(name: "PRODUCED", direction: "OUT") - wrote: [Movie] @relation(name: "WROTE", direction: "OUT") - follows: [Person] @relation(name: "FOLLOWS", direction: "OUT") - reviewed: [Movie] @relation(name: "REVIEWED", direction: "OUT") - REVIEWED_rel: [REVIEWED] -} - -type Movie { - _id: Long! - released: Int! - tagline: String - title: String! - persons_acted_in: [Person] @relation(name: "ACTED_IN", direction: "IN") - persons_directed: [Person] @relation(name: "DIRECTED", direction: "IN") - persons_produced: [Person] @relation(name: "PRODUCED", direction: "IN") - persons_wrote: [Person] @relation(name: "WROTE", direction: "IN") - persons_reviewed: [Person] @relation(name: "REVIEWED", direction: "IN") -} +const typeDefs = gql` + type Person { + _id: Long! + born: Int + name: String! + acted_in: [Movie] @relation(name: "ACTED_IN", direction: "OUT") + ACTED_IN_rel: [ACTED_IN] + directed: [Movie] @relation(name: "DIRECTED", direction: "OUT") + produced: [Movie] @relation(name: "PRODUCED", direction: "OUT") + wrote: [Movie] @relation(name: "WROTE", direction: "OUT") + follows: [Person] @relation(name: "FOLLOWS", direction: "OUT") + reviewed: [Movie] @relation(name: "REVIEWED", direction: "OUT") + REVIEWED_rel: [REVIEWED] + } -type ACTED_IN @relation(name: "ACTED_IN") { - from: Person! - to: Movie! - roles: [String]! -} + type Movie { + _id: Long! + released: Int! + tagline: String + title: String! + persons_acted_in: [Person] @relation(name: "ACTED_IN", direction: "IN") + persons_directed: [Person] @relation(name: "DIRECTED", direction: "IN") + persons_produced: [Person] @relation(name: "PRODUCED", direction: "IN") + persons_wrote: [Person] @relation(name: "WROTE", direction: "IN") + persons_reviewed: [Person] @relation(name: "REVIEWED", direction: "IN") + } -type REVIEWED @relation(name: "REVIEWED") { - from: Person! - to: Movie! - rating: Int! - summary: String! -} + type ACTED_IN @relation(name: "ACTED_IN") { + from: Person! + to: Movie! + roles: [String]! + } + type REVIEWED @relation(name: "REVIEWED") { + from: Person! + to: Movie! + rating: Int! + summary: String! + } `; const schema = makeAugmentedSchema({ typeDefs }); diff --git a/example/apollo-server/interface-union-example.js b/example/apollo-server/interface-union-example.js index 09780ff0..6f589ec7 100644 --- a/example/apollo-server/interface-union-example.js +++ b/example/apollo-server/interface-union-example.js @@ -1,24 +1,28 @@ +import { gql } from 'apollo-server'; import { makeAugmentedSchema } from '../../src/index'; const { ApolloServer } = require('apollo-server'); const neo4j = require('neo4j-driver'); -const __unionTypeDefs = ` -union SearchResult = Blog | Movie +const __unionTypeDefs = gql` + union SearchResult = Blog | Movie -type Blog { - blogId: ID! - created: DateTime - content: String -} + type Blog { + blogId: ID! + created: DateTime + content: String + } -type Movie { - movieId: ID! - title: String -} + type Movie { + movieId: ID! + title: String + } -type Query { - search(searchString: String!): [SearchResult] @cypher(statement:"CALL db.index.fulltext.queryNodes('searchIndex', $searchString) YIELD node RETURN node") -} + type Query { + search(searchString: String!): [SearchResult] + @cypher( + statement: "CALL db.index.fulltext.queryNodes('searchIndex', $searchString) YIELD node RETURN node" + ) + } `; const __interfaceTypeDefs = ` diff --git a/example/apollo-server/movies-schema.js b/example/apollo-server/movies-schema.js index 5c4a5814..de82c05b 100644 --- a/example/apollo-server/movies-schema.js +++ b/example/apollo-server/movies-schema.js @@ -1,143 +1,176 @@ +import { gql } from 'apollo-server'; import { neo4jgraphql } from '../../src/index'; -export const typeDefs = ` -type Movie { - movieId: ID! - title: String - someprefix_title_with_underscores: String - year: Int - dateTime: DateTime - localDateTime: LocalDateTime - date: Date - plot: String - poster: String - imdbRating: Float - ratings: [Rated] - genres: [Genre] @relation(name: "IN_GENRE", direction: "OUT") - similar(first: Int = 3, offset: Int = 0, limit: Int = 5): [Movie] @cypher(statement: "MATCH (this)--(:Genre)--(o:Movie) RETURN o LIMIT $limit") - mostSimilar: Movie @cypher(statement: "RETURN this") - degree: Int @cypher(statement: "RETURN SIZE((this)--())") - actors(first: Int = 3, offset: Int = 0): [Actor] @relation(name: "ACTED_IN", direction:"IN") - avgStars: Float - filmedIn: State @relation(name: "FILMED_IN", direction: "OUT") - location: Point - locations: [Point] - scaleRating(scale: Int = 3): Float @cypher(statement: "RETURN $scale * this.imdbRating") - scaleRatingFloat(scale: Float = 1.5): Float @cypher(statement: "RETURN $scale * this.imdbRating") - _id: ID -} - -type Genre { - name: String - movies(first: Int = 3, offset: Int = 0): [Movie] @relation(name: "IN_GENRE", direction: "IN") - highestRatedMovie: Movie @cypher(statement: "MATCH (m:Movie)-[:IN_GENRE]->(this) RETURN m ORDER BY m.imdbRating DESC LIMIT 1") -} - -type State { - name: String -} - -interface Person { - userId: ID! - name: String -} - -type Actor { - id: ID! - name: String - movies: [Movie] @relation(name: "ACTED_IN", direction: "OUT") - knows: [Person] @relation(name: "KNOWS", direction: "OUT") -} - -type User implements Person { - userId: ID! - name: String - rated: [Rated] -} - -type Rated @relation(name:"RATED") { - from: User - to: Movie - timestamp: Int - date: Date - rating: Float -} -enum BookGenre { - Mystery, - Science, - Math -} - -type OnlyDate { - date: Date -} - -type SpatialNode { - id: ID! - point: Point - spatialNodes(point: Point): [SpatialNode] - @relation(name: "SPATIAL", direction: OUT) -} - -type Book { - genre: BookGenre -} - -interface Camera { - id: ID! - type: String - make: String - weight: Int - operators: [Person] @relation(name: "cameras", direction: IN) - computedOperators(name: String): [Person] @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") -} - -type OldCamera implements Camera { - id: ID! - type: String - make: String - weight: Int - smell: String - operators: [Person] @relation(name: "cameras", direction: IN) - computedOperators(name: String): [Person] @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") -} - -type NewCamera implements Camera { - id: ID! - type: String - make: String - weight: Int - features: [String] - operators: [Person] @relation(name: "cameras", direction: IN) - computedOperators(name: String): [Person] @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") -} - -type CameraMan implements Person { - userId: ID! - name: String - favoriteCamera: Camera @relation(name: "favoriteCamera", direction: "OUT") - heaviestCamera: [Camera] @cypher(statement: "MATCH (c: Camera)--(this) RETURN c ORDER BY c.weight DESC LIMIT 1") - cameras: [Camera!]! @relation(name: "cameras", direction: "OUT") - cameraBuddy: Person @relation(name: "cameraBuddy", direction: "OUT") -} - -union MovieSearch = Movie | Genre | Book | User | OldCamera - -type Query { - Movie(movieId: ID, title: String, year: Int, plot: String, poster: String, imdbRating: Float): [Movie] - MoviesByYear(year: Int, first: Int = 10, offset: Int = 0): [Movie] - AllMovies: [Movie] - MovieById(movieId: ID!): Movie - GenresBySubstring(substring: String): [Genre] @cypher(statement: "MATCH (g:Genre) WHERE toLower(g.name) CONTAINS toLower($substring) RETURN g") - Books: [Book] - CustomCameras: [Camera] @cypher(statement: "MATCH (c:Camera) RETURN c") - CustomCamera: Camera @cypher(statement: "MATCH (c:Camera) RETURN c") -} - -type Mutation { - CustomCamera: Camera @cypher(statement: "CREATE (newCamera:Camera:NewCamera {id: apoc.create.uuid(), type: 'macro'}) RETURN newCamera") - CustomCameras: [Camera] @cypher(statement: "CREATE (newCamera:Camera:NewCamera {id: apoc.create.uuid(), type: 'macro', features: ['selfie', 'zoom']}) CREATE (oldCamera:Camera:OldCamera {id: apoc.create.uuid(), type: 'floating', smell: 'rusty' }) RETURN [newCamera, oldCamera]") -} +export const typeDefs = gql` + type Movie { + movieId: ID! + title: String + someprefix_title_with_underscores: String + year: Int + dateTime: DateTime + localDateTime: LocalDateTime + date: Date + plot: String + poster: String + imdbRating: Float + ratings: [Rated] + genres: [Genre] @relation(name: "IN_GENRE", direction: "OUT") + similar(first: Int = 3, offset: Int = 0, limit: Int = 5): [Movie] + @cypher( + statement: "MATCH (this)--(:Genre)--(o:Movie) RETURN o LIMIT $limit" + ) + mostSimilar: Movie @cypher(statement: "RETURN this") + degree: Int @cypher(statement: "RETURN SIZE((this)--())") + actors(first: Int = 3, offset: Int = 0): [Actor] + @relation(name: "ACTED_IN", direction: "IN") + avgStars: Float + filmedIn: State @relation(name: "FILMED_IN", direction: "OUT") + location: Point + locations: [Point] + scaleRating(scale: Int = 3): Float + @cypher(statement: "RETURN $scale * this.imdbRating") + scaleRatingFloat(scale: Float = 1.5): Float + @cypher(statement: "RETURN $scale * this.imdbRating") + _id: ID + } + + type Genre { + name: String + movies(first: Int = 3, offset: Int = 0): [Movie] + @relation(name: "IN_GENRE", direction: "IN") + highestRatedMovie: Movie + @cypher( + statement: "MATCH (m:Movie)-[:IN_GENRE]->(this) RETURN m ORDER BY m.imdbRating DESC LIMIT 1" + ) + } + + type State { + name: String + } + + interface Person { + userId: ID! + name: String + } + + type Actor { + id: ID! + name: String + movies: [Movie] @relation(name: "ACTED_IN", direction: "OUT") + knows: [Person] @relation(name: "KNOWS", direction: "OUT") + } + + type User implements Person { + userId: ID! + name: String + rated: [Rated] + } + + type Rated @relation(name: "RATED") { + from: User + to: Movie + timestamp: Int + date: Date + rating: Float + } + enum BookGenre { + Mystery + Science + Math + } + + type OnlyDate { + date: Date + } + + type SpatialNode { + id: ID! + point: Point + spatialNodes(point: Point): [SpatialNode] + @relation(name: "SPATIAL", direction: OUT) + } + + type Book { + genre: BookGenre + } + + interface Camera { + id: ID! + type: String + make: String + weight: Int + operators: [Person] @relation(name: "cameras", direction: IN) + computedOperators(name: String): [Person] + @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") + } + + type OldCamera implements Camera { + id: ID! + type: String + make: String + weight: Int + smell: String + operators: [Person] @relation(name: "cameras", direction: IN) + computedOperators(name: String): [Person] + @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") + } + + type NewCamera implements Camera { + id: ID! + type: String + make: String + weight: Int + features: [String] + operators: [Person] @relation(name: "cameras", direction: IN) + computedOperators(name: String): [Person] + @cypher(statement: "MATCH (this)<-[:cameras]-(p:Person) RETURN p") + } + + type CameraMan implements Person { + userId: ID! + name: String + favoriteCamera: Camera @relation(name: "favoriteCamera", direction: "OUT") + heaviestCamera: [Camera] + @cypher( + statement: "MATCH (c: Camera)--(this) RETURN c ORDER BY c.weight DESC LIMIT 1" + ) + cameras: [Camera!]! @relation(name: "cameras", direction: "OUT") + cameraBuddy: Person @relation(name: "cameraBuddy", direction: "OUT") + } + + union MovieSearch = Movie | Genre | Book | User | OldCamera + + type Query { + Movie( + movieId: ID + title: String + year: Int + plot: String + poster: String + imdbRating: Float + ): [Movie] + MoviesByYear(year: Int, first: Int = 10, offset: Int = 0): [Movie] + AllMovies: [Movie] + MovieById(movieId: ID!): Movie + GenresBySubstring(substring: String): [Genre] + @cypher( + statement: "MATCH (g:Genre) WHERE toLower(g.name) CONTAINS toLower($substring) RETURN g" + ) + Books: [Book] + CustomCameras: [Camera] @cypher(statement: "MATCH (c:Camera) RETURN c") + CustomCamera: Camera @cypher(statement: "MATCH (c:Camera) RETURN c") + } + + type Mutation { + CustomCamera: Camera + @cypher( + statement: "CREATE (newCamera:Camera:NewCamera {id: apoc.create.uuid(), type: 'macro'}) RETURN newCamera" + ) + CustomCameras: [Camera] + @cypher( + statement: "CREATE (newCamera:Camera:NewCamera {id: apoc.create.uuid(), type: 'macro', features: ['selfie', 'zoom']}) CREATE (oldCamera:Camera:OldCamera {id: apoc.create.uuid(), type: 'floating', smell: 'rusty' }) RETURN [newCamera, oldCamera]" + ) + } `; export const resolvers = { diff --git a/example/apollo-server/movies-typedefs.js b/example/apollo-server/movies-typedefs.js index aac3d43f..cf77d405 100644 --- a/example/apollo-server/movies-typedefs.js +++ b/example/apollo-server/movies-typedefs.js @@ -1,47 +1,48 @@ +import { gql } from 'apollo-server'; import { makeAugmentedSchema } from '../../src/index'; import { ApolloServer } from 'apollo-server'; import { v1 as neo4j } from 'neo4j-driver'; -const typeDefs = ` -type Movie { - movieId: ID! - title: String - year: Int - plot: String - poster: String - imdbRating: Float - ratings: [Rated] - genres: [Genre] @relation(name: "IN_GENRE", direction: "OUT") - actors: [Actor] @relation(name: "ACTED_IN", direction: "IN") -} -type Genre { - name: String - movies: [Movie] @relation(name: "IN_GENRE", direction: "IN") -} -type Actor { - id: ID! - name: String - movies: [Movie] @relation(name: "ACTED_IN", direction: "OUT") -} -type User { - userId: ID! - name: String - rated: [Rated] - recommendedMovies: [Movie] - @cypher( - statement: """ - MATCH (this)-[:RATED]->(:Movie)<-[:RATED]-(:User)-[:RATED]->(rec:Movie) - WITH rec, COUNT(*) AS score ORDER BY score DESC LIMIT 10 - RETURN rec - """ - ) -} -type Rated @relation(name: "RATED") { - from: User - to: Movie - rating: Float - created: DateTime -} +const typeDefs = gql` + type Movie { + movieId: ID! + title: String + year: Int + plot: String + poster: String + imdbRating: Float + ratings: [Rated] + genres: [Genre] @relation(name: "IN_GENRE", direction: "OUT") + actors: [Actor] @relation(name: "ACTED_IN", direction: "IN") + } + type Genre { + name: String + movies: [Movie] @relation(name: "IN_GENRE", direction: "IN") + } + type Actor { + id: ID! + name: String + movies: [Movie] @relation(name: "ACTED_IN", direction: "OUT") + } + type User { + userId: ID! + name: String + rated: [Rated] + recommendedMovies: [Movie] + @cypher( + statement: """ + MATCH (this)-[:RATED]->(:Movie)<-[:RATED]-(:User)-[:RATED]->(rec:Movie) + WITH rec, COUNT(*) AS score ORDER BY score DESC LIMIT 10 + RETURN rec + """ + ) + } + type Rated @relation(name: "RATED") { + from: User + to: Movie + rating: Float + created: DateTime + } `; const schema = makeAugmentedSchema({ typeDefs }); diff --git a/example/apollo-server/multidatabase.js b/example/apollo-server/multidatabase.js index 5ff6e374..16e19d6c 100644 --- a/example/apollo-server/multidatabase.js +++ b/example/apollo-server/multidatabase.js @@ -1,10 +1,10 @@ +import { gql } from 'apollo-server'; import { makeAugmentedSchema } from '../../src/index'; import { ApolloServer } from 'apollo-server'; import neo4j from 'neo4j-driver'; -const typeDefs = ` - -type User { +const typeDefs = gql` + type User { name: String! wrote: [Review] @relation(name: "WROTE", direction: "OUT") } diff --git a/package-lock.json b/package-lock.json index c35a5dc3..8979cbb5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "neo4j-graphql-js", - "version": "2.19.0", + "version": "2.19.1", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4877,6 +4877,11 @@ "is-obj": "^2.0.0" } }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, "duplexer3": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", diff --git a/package.json b/package.json index eb0d8479..565271d4 100755 --- a/package.json +++ b/package.json @@ -1,9 +1,15 @@ { "name": "neo4j-graphql-js", - "version": "2.19.1", + "version": "2.19.0", "description": "A GraphQL to Cypher query execution layer for Neo4j. ", "main": "./dist/index.js", "scripts": { + "reinstall": "sudo rm neo4j -r && npm install", + "preinstall": "sudo bash ./scripts/install-neo4j.sh", + "neo4j:start": "sudo bash ./scripts/start-neo4j.sh", + "neo4j:stop": "sudo bash ./scripts/stop-and-clear-neo4j.sh", + "prestart": "npm run neo4j:start", + "prestart-gateway": "npm run neo4j:start", "start": "nodemon ./example/apollo-server/movies.js --exec babel-node -e js", "autogen": "nodemon ./example/autogenerated/autogen.js --exec babel-node -e js", "start-middleware": "nodemon ./example/apollo-server/movies-middleware.js --exec babel-node -e js", @@ -71,6 +77,7 @@ "@babel/runtime-corejs2": "^7.5.5", "apollo-server-errors": "^2.4.1", "debug": "^4.1.1", + "dotenv": "^8.2.0", "graphql-auth-directives": "^2.2.1", "lodash": "^4.17.19", "neo4j-driver": "^4.2.1", diff --git a/scripts/helpers/cache.sh b/scripts/helpers/cache.sh new file mode 100644 index 00000000..2cf910b1 --- /dev/null +++ b/scripts/helpers/cache.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +get_latest_download() { + local cache="$1" # Save first argument in a variable + local filename_glob="$2" + cd $cache + { # try + latest_download="$(ls -r $filename_glob | head -1)" + #save your output + } 2>/dev/null || { + # catch + # save log for exception + latest_download=() + } + cd .. +} + +check_if_there_is_need_to_download() { + local cache="$1" # Save first argument in a variable + local file_URL="$2" # Save first argument in a variable + local filename="$3" # Save first argument in a variable + get_latest_download $cache $filename* + + file_to_download=() + if [ ! "$latest_download" ]; then + file_to_download=($file_URL/$filename) + fi +} +get_list_of_files_to_download() { + # $1 cache folder + # $2..n files to check if available in cache + local cache="$1" # Save first argument in a variable + shift # Shift all arguments to the left (original $1 gets lost) + local files_info=("$@") # Rebuild the array with rest of arguments + local pattern='(.*):+(.*)' + + files_to_download=() + + for file_info in "${files_info[@]}"; do + [[ $file_info =~ $pattern ]] + check_if_there_is_need_to_download $cache ${BASH_REMATCH[1]} ${BASH_REMATCH[2]} + files_to_download+=($file_to_download) + done +} diff --git a/scripts/helpers/cached_download.sh b/scripts/helpers/cached_download.sh new file mode 100644 index 00000000..e5d0a73b --- /dev/null +++ b/scripts/helpers/cached_download.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +source=${BASH_SOURCE[0]} + +cached_download() { + . $(dirname $source)/get_source_dir.sh + local this_directory=$(get_source_dir $source) + . $this_directory/cache.sh + . $this_directory/download_files.sh + . $this_directory/delete_duplicates.sh + # $1 download_info + # $2 cache_folder + local cache="$1" # Save first argument in a variable + shift # Shift all arguments to the left (original $1 gets lost) + local files_info=("$@") # Rebuild the array with rest of arguments + get_list_of_files_to_download $cache "${files_info[@]}" + download_files $cache "${files_to_download[@]}" + delete_duplicates $cache # just in case + cached_downloads=() + local pattern='(.*):+(.*)' + for file_info in ${files_info[@]}; do + [[ $file_info =~ $pattern ]] + local file_name=${BASH_REMATCH[2]} + get_latest_download $cache $file_name + cached_downloads+=($cache/$latest_download) + done +} diff --git a/scripts/helpers/delete_duplicates.sh b/scripts/helpers/delete_duplicates.sh new file mode 100644 index 00000000..7c47f68e --- /dev/null +++ b/scripts/helpers/delete_duplicates.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# sudo apt-get install fdupes + +delete_duplicates() { + local folder_to_look_for_duplicates="$1" + fdupes -rdN $folder_to_look_for_duplicates +} diff --git a/scripts/helpers/download_files.sh b/scripts/helpers/download_files.sh new file mode 100644 index 00000000..4b7349e3 --- /dev/null +++ b/scripts/helpers/download_files.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source=${BASH_SOURCE[0]} +. $(dirname $source)/get_source_dir.sh + +download_files() { + local this_directory=$(get_source_dir $source) + local cache="$1" # Save first argument in a variable + shift # Shift all arguments to the left (original $1 gets lost) + local files_to_download=("$@") # Rebuild the array with rest of arguments + local number_of_files_to_download=${#files_to_download[@]} + + if [ $number_of_files_to_download -gt 0 ]; then + . $this_directory/get_number_of_logical_processors.sh + get_number_of_logical_processors + echo ${files_to_download[@]} | xargs -n 1 -P $logical_cpus wget -q -P $cache --show-progress + fi +} diff --git a/scripts/helpers/execute_start.sh b/scripts/helpers/execute_start.sh new file mode 100644 index 00000000..02ac1795 --- /dev/null +++ b/scripts/helpers/execute_start.sh @@ -0,0 +1,34 @@ +#!/bin/dash + +this_directory=$1 +localhost=$2 + +. $this_directory/helpers/load_env_vars.sh +load_env_vars + +if [ ! -d "neo4j/data/databases/graph.db" ]; then + echo "Neo4j not installed correctly, run ./scripts/install_neo4j" + exit 1 +else + echo "dbms.allow_upgrade=true" >>./neo4j/conf/neo4j.conf + echo "dbms.recovery.fail_on_missing_files=false" >>./neo4j/conf/neo4j.conf + # Set initial and max heap to workaround JVM in docker issues + dbms_memory_heap_initial_size="2048m" dbms_memory_heap_max_size="2048m" ./neo4j/bin/neo4j start + echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)" + + echo "Endpoint is $localhost:$BOLT_PORT" + for i in {1..120}; do + nc -z $localhost $BOLT_PORT -w 2 + is_up=$? + if [ $is_up -eq 0 ]; then + echo + echo "Successfully started, neo4j bolt available on $BOLT_PORT" + break + fi + sleep 1 + echo -n "." + done + echo + # Wait a further 5 seconds after the port is available + sleep 5 +fi diff --git a/scripts/helpers/execute_stop.sh b/scripts/helpers/execute_stop.sh new file mode 100644 index 00000000..c896d4e8 --- /dev/null +++ b/scripts/helpers/execute_stop.sh @@ -0,0 +1,28 @@ +#!/bin/dash + +this_directory=$1 +localhost=$2 + +. $this_directory/helpers/load_env_vars.sh +load_env_vars + +./neo4j/bin/neo4j stop +rm -r neo4j/data/databases/graph.db +./neo4j/bin/neo4j start + +echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)" + +for i in {1..120}; do + nc -z $localhost $BOLT_PORT -w 2 + is_up=$? + if [ $is_up -eq 0 ]; then + echo + echo "Successfully started, neo4j bolt available on $BOLT_PORT" + break + fi + sleep 1 + echo -n "." +done +echo +# Wait a further 5 seconds after the port is available +sleep 5 diff --git a/scripts/helpers/execute_wait.sh b/scripts/helpers/execute_wait.sh new file mode 100644 index 00000000..cf1a8328 --- /dev/null +++ b/scripts/helpers/execute_wait.sh @@ -0,0 +1,23 @@ +#!/bin/dash + +this_directory=$1 +localhost=$2 + +. $this_directory/helpers/load_env_vars.sh +load_env_vars + +echo "Waiting up to 2 minutes for graphql http port ($HTTP_PORT)" + +. $this_directory/helpers/get_local_host.sh +for i in {1..120}; do + nc -z $(get_local_host) $HTTP_PORT -w 2 + is_up=$? + if [ $is_up -eq 0 ]; then + echo + echo "Successfully started, graphql http available on $HTTP_PORT" + break + fi + sleep 1 + echo -n "." +done +echo diff --git a/scripts/helpers/get_local_host.sh b/scripts/helpers/get_local_host.sh new file mode 100644 index 00000000..8e8c1970 --- /dev/null +++ b/scripts/helpers/get_local_host.sh @@ -0,0 +1,10 @@ +#!/bin/bash +source=${BASH_SOURCE[0]} +. $(dirname $source)/get_source_dir.sh + +get_local_host() { + local this_directory=$(get_source_dir $source) + local localhost=$(perl $this_directory/regex.pl "$(cat /etc/resolv.conf)") + echo $localhost +} +# echo "$(get_local_host)" diff --git a/scripts/helpers/get_number_of_logical_processors.sh b/scripts/helpers/get_number_of_logical_processors.sh new file mode 100644 index 00000000..ff161464 --- /dev/null +++ b/scripts/helpers/get_number_of_logical_processors.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +get_number_of_logical_processors() { + logical_cpus=$([ $(uname) = 'Darwin' ] && + sysctl -n hw.logicalcpu_max || + lscpu -p | egrep -v '^#' | wc -l) +} diff --git a/scripts/helpers/get_source_dir.sh b/scripts/helpers/get_source_dir.sh new file mode 100644 index 00000000..6ccbc8b7 --- /dev/null +++ b/scripts/helpers/get_source_dir.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Based on: +# https://stackoverflow.com/questions/59895/how-can-i-get-the-source-directory-of-a-bash-script-from-within-the-script-itsel + +get_source_dir() { + local this_source=$1 + while [ -h "$this_source" ]; do # resolve $this_source until the file is no longer a symlink + local this_dir="$(cd -P "$(dirname "$this_source")" >/dev/null 2>&1 && pwd)" + this_source="$(readlink "$this_source")" + [[ $this_source != /* ]] && this_source="$this_dir/$this_source" # if $this_source was a relative symlink, we need to resolve it relative to the path where the symlink file was located + done + this_dir="$(cd -P "$(dirname "$this_source")" >/dev/null 2>&1 && pwd)" + echo $this_dir +} diff --git a/scripts/helpers/load_env_vars.sh b/scripts/helpers/load_env_vars.sh new file mode 100644 index 00000000..86a9b9c5 --- /dev/null +++ b/scripts/helpers/load_env_vars.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +load_env_vars() { + if [ ! -f ../.env ]; then + export $(cat .env | xargs) + fi +} diff --git a/scripts/helpers/regex.pl b/scripts/helpers/regex.pl new file mode 100644 index 00000000..cfddb192 --- /dev/null +++ b/scripts/helpers/regex.pl @@ -0,0 +1,14 @@ +use strict; +my $regex = qr/(?<=nameserver )(?:[0-9]{1,3}\.){3}[0-9]{1,3}/p; + + +if ( $ARGV[0] =~ /$regex/ ) { + print "${^MATCH}"; + + # print "Whole match is ${^MATCH} and its start/end positions can be obtained via \$-[0] and \$+[0]\n"; + # print "Capture Group 1 is $1 and its start/end positions can be obtained via \$-[1] and \$+[1]\n"; + # print "Capture Group 2 is $2 ... and so on\n"; +} + +# ${^POSTMATCH} and ${^PREMATCH} are also available with the use of '/p' +# Named capture groups can be called via $+{name} \ No newline at end of file diff --git a/scripts/helpers/test_get_source_data.sh b/scripts/helpers/test_get_source_data.sh new file mode 100644 index 00000000..44545cde --- /dev/null +++ b/scripts/helpers/test_get_source_data.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +source=${BASH_SOURCE[0]} +. $(dirname $source)/get_source_dir.sh +this_directory=$(get_source_dir $source) +echo $this_directory diff --git a/scripts/install-neo4j.sh b/scripts/install-neo4j.sh old mode 100755 new mode 100644 index e58ea6b8..805b269c --- a/scripts/install-neo4j.sh +++ b/scripts/install-neo4j.sh @@ -1,17 +1,49 @@ -#!/usr/bin/env bash - -set -xe - -if [ ! -d "neo4j/data/databases/graph.db" ]; then - mkdir -p neo4j - wget dist.neo4j.org/neo4j-$NEO4J_DIST-$NEO4J_VERSION-unix.tar.gz - tar -xzf neo4j-$NEO4J_DIST-$NEO4J_VERSION-unix.tar.gz -C neo4j --strip-components 1 - neo4j/bin/neo4j-admin set-initial-password letmein - curl -L https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/$APOC_VERSION/apoc-$APOC_VERSION-all.jar > ./neo4j/plugins/apoc-$APOC_VERSION-all.jar - wget https://datastores.s3.amazonaws.com/recommendations/v$DATASTORE_VERSION/recommendations.db.zip - sudo apt-get install unzip - unzip recommendations.db.zip - mv recommendations.db neo4j/data/databases/graph.db -else +#!/bin/bash + +source=${BASH_SOURCE[0]} +. $(dirname $source)/helpers/get_source_dir.sh + +install-neo4j() { + local this_directory=$(get_source_dir $source) + + yes | sudo apt-get install fdupes unzip + + cache=.download_cache + helpers_path=$this_directory/helpers + . $helpers_path/load_env_vars.sh + load_env_vars + + # set -xe + + graph_db_path="neo4j/data/databases/graph.db" + if [ ! -d $graph_db_path ]; then + if [ ! -L neo4j ]; then + mkdir -p -- neo4j + mkdir -p -- $cache + fi + neo4j_URL=dist.neo4j.org + apoc_URL=https://github.com/neo4j-contrib/neo4j-apoc-procedures/releases/download/$APOC_VERSION + recommendations_URL=https://datastores.s3.amazonaws.com/recommendations/v$DATASTORE_VERSION + neo4j=neo4j-$NEO4J_DIST-$NEO4J_VERSION-unix.tar.gz + apoc=apoc-$APOC_VERSION-all.jar + recommendations=recommendations.db.zip + + download_info=("$neo4j_URL:$neo4j" "$apoc_URL:$apoc" "$recommendations_URL:$recommendations") + + . $helpers_path/cached_download.sh + cached_download $cache ${download_info[@]} + + tar -xzf ${cached_downloads[0]} -C neo4j --strip-components 1 + cp ${cached_downloads[1]} neo4j/plugins/$apoc + unzip -o ${cached_downloads[2]} + + mv recommendations.db $graph_db_path + rm __MACOSX* -r + + neo4j/bin/neo4j-admin set-default-admin $NEO4J_USER + neo4j/bin/neo4j-admin set-initial-password $NEO4J_PASSWORD + else echo "Database is already installed, skipping" -fi + fi +} +install-neo4j diff --git a/scripts/start-neo4j.sh b/scripts/start-neo4j.sh index 06ffb68d..425c978c 100755 --- a/scripts/start-neo4j.sh +++ b/scripts/start-neo4j.sh @@ -1,30 +1,13 @@ -#!/usr/bin/env bash +#!/bin/bash -BOLT_PORT=7687 +source=${BASH_SOURCE[0]} +. $(dirname $source)/helpers/get_source_dir.sh -if [ ! -d "neo4j/data/databases/graph.db" ]; then - echo "Neo4j not installed correctly, run ./scripts/install_neo4j" - exit 1 -else - echo "dbms.allow_upgrade=true" >> ./neo4j/conf/neo4j.conf - echo "dbms.recovery.fail_on_missing_files=false" >> ./neo4j/conf/neo4j.conf - # Set initial and max heap to workaround JVM in docker issues - dbms_memory_heap_initial_size="2048m" dbms_memory_heap_max_size="2048m" ./neo4j/bin/neo4j start - echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)" +start-neo4j() { + local this_directory=$(get_source_dir $source) - for i in {1..120}; - do - nc -z 127.0.0.1 $BOLT_PORT - is_up=$? - if [ $is_up -eq 0 ]; then - echo - echo "Successfully started, neo4j bolt available on $BOLT_PORT" - break - fi - sleep 1 - echo -n "." - done - echo - # Wait a further 5 seconds after the port is available - sleep 5 -fi \ No newline at end of file + . $this_directory/helpers/get_local_host.sh + localhost=$(get_local_host) + dash $this_directory/helpers/execute_start.sh $this_directory $localhost +} +start-neo4j diff --git a/scripts/stop-and-clear-neo4j.sh b/scripts/stop-and-clear-neo4j.sh index 3e151af4..7166c622 100755 --- a/scripts/stop-and-clear-neo4j.sh +++ b/scripts/stop-and-clear-neo4j.sh @@ -1,25 +1,13 @@ -#!/usr/bin/env bash +#!/bin/bash -BOLT_PORT=7687 +source=${BASH_SOURCE[0]} +. $(dirname $source)/helpers/get_source_dir.sh -./neo4j/bin/neo4j stop -rm -r neo4j/data/databases/graph.db -./neo4j/bin/neo4j start +stop-and-clear-neo4j() { + local this_directory=$(get_source_dir $source) -echo "Waiting up to 2 minutes for neo4j bolt port ($BOLT_PORT)" - -for i in {1..120}; - do - nc -z 127.0.0.1 $BOLT_PORT - is_up=$? - if [ $is_up -eq 0 ]; then - echo - echo "Successfully started, neo4j bolt available on $BOLT_PORT" - break - fi - sleep 1 - echo -n "." -done -echo -# Wait a further 5 seconds after the port is available -sleep 5 \ No newline at end of file + . $this_directory/helpers/get_local_host.sh + localhost=$(get_local_host) + dash $this_directory/helpers/execute_stop.sh $this_directory $localhost +} +stop-and-clear-neo4j diff --git a/scripts/wait-for-graphql.sh b/scripts/wait-for-graphql.sh index 6b5373b1..6084ac0d 100755 --- a/scripts/wait-for-graphql.sh +++ b/scripts/wait-for-graphql.sh @@ -1,19 +1,13 @@ -#!/usr/bin/env bash +#!/bin/bash -HTTP_PORT=3000 +source=${BASH_SOURCE[0]} +. $(dirname $source)/helpers/get_source_dir.sh -echo "Waiting up to 2 minutes for graphql http port ($HTTP_PORT)" +wait-for-graphql() { + local this_directory=$(get_source_dir $source) -for i in {1..120}; - do - nc -z localhost $HTTP_PORT - is_up=$? - if [ $is_up -eq 0 ]; then - echo - echo "Successfully started, graphql http available on $HTTP_PORT" - break - fi - sleep 1 - echo -n "." -done -echo \ No newline at end of file + . $this_directory/helpers/get_local_host.sh + localhost=$(get_local_host) + dash $this_directory/helpers/execute_wait.sh $this_directory $localhost +} +wait-for-graphql