From 5d3fe409d3a6f2f75781ebe387e4000606517772 Mon Sep 17 00:00:00 2001 From: Fabio Buso Date: Tue, 19 Jul 2022 18:48:24 +0200 Subject: [PATCH] [HOPSWORKS-3256] Remove old feature store tour job files (#1028) --- .../common/project/ProjectController.java | 64 ++----------------- .../hops/hopsworks/common/util/Settings.java | 5 -- 2 files changed, 6 insertions(+), 63 deletions(-) diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java index 032c45bf5e..43d83c7ad4 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java @@ -132,8 +132,6 @@ import io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers; import io.hops.hopsworks.persistence.entity.jobs.configuration.JobConfiguration; import io.hops.hopsworks.persistence.entity.jobs.configuration.JobType; -import io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration; -import io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO; import io.hops.hopsworks.persistence.entity.jobs.description.Jobs; import io.hops.hopsworks.persistence.entity.jobs.quota.YarnPriceMultiplicator; import io.hops.hopsworks.persistence.entity.jobs.quota.YarnProjectsQuota; @@ -2530,42 +2528,16 @@ public String addTourFilesToProject(String username, Project project, Distribute } break; case FS: - datasetController.createDataset(user, project, tourFilesDataset, "files for guide projects", - Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET), - false, DatasetAccessPermission.EDITABLE, dfso); - // Get the JAR from /user/ - String featurestoreExampleJarSrc = "/user/" + settings.getSparkUser() + "/" - + settings.getHopsExamplesFeaturestoreTourFilename(); - String featurestoreExampleJarDst = projectPath - + tourFilesDataset + "/" + settings.getHopsExamplesFeaturestoreTourFilename(); - // Get the sample data and notebooks from /user//featurestore_demo/ - String featurestoreExampleDataSrc = "/user/" + settings.getHdfsSuperUser() + "/" + - Settings.HOPS_FEATURESTORE_TOUR_DATA + "/data"; - String featurestoreExampleDataDst = projectPath + tourFilesDataset; - try { - //Move example .jar file to HDFS - udfso.copyInHdfs(new Path(featurestoreExampleJarSrc), new Path(featurestoreExampleJarDst)); - String datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset); - String userHdfsName = hdfsUsersController.getHdfsUserName(project, user); - udfso.setPermission(new Path(featurestoreExampleJarDst), - udfso.getParentPermission(new Path(featurestoreExampleJarDst))); - udfso.setOwner(new Path(featurestoreExampleJarDst), userHdfsName, datasetGroup); //Move example data and notebooks to HDFS - udfso.copyInHdfs(new Path(featurestoreExampleDataSrc), new Path(featurestoreExampleDataDst)); - datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset); - userHdfsName = hdfsUsersController.getHdfsUserName(project, user); - Inode featurestoreDataDst = inodeController.getInodeAtPath(featurestoreExampleDataDst); - datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleDataDst), - FsPermission.createImmutable(featurestoreDataDst.getPermission()), - userHdfsName, datasetGroup, dfso, udfso); + String userHdfsName = hdfsUsersController.getHdfsUserName(project, user); + String datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER); //Move example notebooks to Jupyter dataset - String featurestoreExampleNotebooksSrc = "/user/" + settings.getHdfsSuperUser() + "/" + - Settings.HOPS_FEATURESTORE_TOUR_DATA + "/notebooks"; + String featurestoreExampleNotebooksSrc = + "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_FEATURESTORE_TOUR_DATA + "/notebooks"; String featurestoreExampleNotebooksDst = projectPath + Settings.HOPS_TOUR_DATASET_JUPYTER; - udfso.copyInHdfs(new Path(featurestoreExampleNotebooksSrc + "/*"), - new Path(featurestoreExampleNotebooksDst)); - datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER); + udfso.copyInHdfs( + new Path(featurestoreExampleNotebooksSrc + "/*"), new Path(featurestoreExampleNotebooksDst)); Inode featurestoreNotebooksDst = inodeController.getInodeAtPath(featurestoreExampleNotebooksDst); datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleNotebooksDst), FsPermission.createImmutable(featurestoreNotebooksDst.getPermission()), @@ -2574,30 +2546,6 @@ public String addTourFilesToProject(String username, Project project, Distribute throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE, "project: " + project.getName(), ex.getMessage(), ex); } - SparkJobConfiguration sparkJobConfiguration = new SparkJobConfiguration(); - sparkJobConfiguration.setAmQueue("default"); - sparkJobConfiguration.setAmMemory(1024); - sparkJobConfiguration.setAmVCores(1); - sparkJobConfiguration.setAppPath("hdfs://" + featurestoreExampleJarDst); - sparkJobConfiguration.setMainClass(Settings.HOPS_FEATURESTORE_TOUR_JOB_CLASS); - sparkJobConfiguration.setDefaultArgs("--input TestJob/data"); - sparkJobConfiguration.setExecutorInstances(1); - sparkJobConfiguration.setExecutorCores(1); - sparkJobConfiguration.setExecutorMemory(2024); - sparkJobConfiguration.setExecutorGpus(0); - sparkJobConfiguration.setDynamicAllocationEnabled(true); - sparkJobConfiguration.setDynamicAllocationMinExecutors(1); - sparkJobConfiguration.setDynamicAllocationMaxExecutors(3); - sparkJobConfiguration.setDynamicAllocationInitialExecutors(1); - sparkJobConfiguration.setAppName(Settings.HOPS_FEATURESTORE_TOUR_JOB_NAME); - sparkJobConfiguration.setLocalResources(new LocalResourceDTO[0]); - Jobs job = jobController.putJob(user, project, null, sparkJobConfiguration); - activityFacade.persistActivity(ActivityFacade.CREATED_JOB + job.getName(), project, user, - ActivityFlag.SERVICE); - executionController - .start(job, Settings.HOPS_FEATURESTORE_TOUR_JOB_INPUT_PARAM + tourFilesDataset + "/data", user); - activityFacade.persistActivity(ActivityFacade.RAN_JOB + job.getName(), project, user, - ActivityFlag.SERVICE); break; default: break; diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java index 9a4fe73c99..83bee5d4b7 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java @@ -2470,11 +2470,6 @@ public synchronized String getHopsExamplesSparkFilename() { return "hops-examples-spark-" + HOPS_EXAMPLES_VERSION + ".jar"; } - public synchronized String getHopsExamplesFeaturestoreTourFilename() { - checkCache(); - return "hops-examples-featurestore-tour-" + HOPS_EXAMPLES_VERSION + ".jar"; - } - private String VERIFICATION_PATH = "/hopsworks-admin/security/validate_account.xhtml"; public synchronized String getEmailVerificationEndpoint() {