[HOPSWORKS-3256] Remove old feature store tour job files (#1028)

logicalclocks · Jul 19, 2022 · 5d3fe40 · 5d3fe40
1 parent d47c6ec
commit 5d3fe40
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 63 deletions.
diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/project/ProjectController.java
@@ -132,8 +132,6 @@
 import io.hops.hopsworks.persistence.entity.hdfs.user.HdfsUsers;
 import io.hops.hopsworks.persistence.entity.jobs.configuration.JobConfiguration;
 import io.hops.hopsworks.persistence.entity.jobs.configuration.JobType;
-import io.hops.hopsworks.persistence.entity.jobs.configuration.spark.SparkJobConfiguration;
-import io.hops.hopsworks.persistence.entity.jobs.configuration.yarn.LocalResourceDTO;
 import io.hops.hopsworks.persistence.entity.jobs.description.Jobs;
 import io.hops.hopsworks.persistence.entity.jobs.quota.YarnPriceMultiplicator;
 import io.hops.hopsworks.persistence.entity.jobs.quota.YarnProjectsQuota;
@@ -2530,42 +2528,16 @@ public String addTourFilesToProject(String username, Project project, Distribute
  }
  break;
  case FS:
- datasetController.createDataset(user, project, tourFilesDataset, "files for guide projects",
- Provenance.getDatasetProvCore(projectProvCore, Provenance.MLType.DATASET),
- false, DatasetAccessPermission.EDITABLE, dfso);
- // Get the JAR from /user/<super user>
- String featurestoreExampleJarSrc = "/user/" + settings.getSparkUser() + "/"
- + settings.getHopsExamplesFeaturestoreTourFilename();
- String featurestoreExampleJarDst = projectPath
- + tourFilesDataset + "/" + settings.getHopsExamplesFeaturestoreTourFilename();
- // Get the sample data and notebooks from /user/<super user>/featurestore_demo/
- String featurestoreExampleDataSrc = "/user/" + settings.getHdfsSuperUser() + "/" +
- Settings.HOPS_FEATURESTORE_TOUR_DATA + "/data";
- String featurestoreExampleDataDst = projectPath + tourFilesDataset;
-
  try {
- //Move example .jar file to HDFS
- udfso.copyInHdfs(new Path(featurestoreExampleJarSrc), new Path(featurestoreExampleJarDst));
- String datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
- String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
- udfso.setPermission(new Path(featurestoreExampleJarDst),
- udfso.getParentPermission(new Path(featurestoreExampleJarDst)));
- udfso.setOwner(new Path(featurestoreExampleJarDst), userHdfsName, datasetGroup);
  //Move example data and notebooks to HDFS
- udfso.copyInHdfs(new Path(featurestoreExampleDataSrc), new Path(featurestoreExampleDataDst));
- datasetGroup = hdfsUsersController.getHdfsGroupName(project, tourFilesDataset);
- userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
- Inode featurestoreDataDst = inodeController.getInodeAtPath(featurestoreExampleDataDst);
- datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleDataDst),
- FsPermission.createImmutable(featurestoreDataDst.getPermission()),
- userHdfsName, datasetGroup, dfso, udfso);
+ String userHdfsName = hdfsUsersController.getHdfsUserName(project, user);
+ String datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER);
  //Move example notebooks to Jupyter dataset
- String featurestoreExampleNotebooksSrc = "/user/" + settings.getHdfsSuperUser() + "/" +
- Settings.HOPS_FEATURESTORE_TOUR_DATA + "/notebooks";
+ String featurestoreExampleNotebooksSrc =
+  "/user/" + settings.getHdfsSuperUser() + "/" + Settings.HOPS_FEATURESTORE_TOUR_DATA + "/notebooks";
  String featurestoreExampleNotebooksDst = projectPath + Settings.HOPS_TOUR_DATASET_JUPYTER;
- udfso.copyInHdfs(new Path(featurestoreExampleNotebooksSrc + "/*"),
- new Path(featurestoreExampleNotebooksDst));
- datasetGroup = hdfsUsersController.getHdfsGroupName(project, Settings.HOPS_TOUR_DATASET_JUPYTER);
+ udfso.copyInHdfs(
+ new Path(featurestoreExampleNotebooksSrc + "/*"), new Path(featurestoreExampleNotebooksDst));
  Inode featurestoreNotebooksDst = inodeController.getInodeAtPath(featurestoreExampleNotebooksDst);
  datasetController.recChangeOwnershipAndPermission(new Path(featurestoreExampleNotebooksDst),
  FsPermission.createImmutable(featurestoreNotebooksDst.getPermission()),
@@ -2574,30 +2546,6 @@ public String addTourFilesToProject(String username, Project project, Distribute
  throw new ProjectException(RESTCodes.ProjectErrorCode.PROJECT_TOUR_FILES_ERROR, Level.SEVERE,
  "project: " + project.getName(), ex.getMessage(), ex);
  }
- SparkJobConfiguration sparkJobConfiguration = new SparkJobConfiguration();
- sparkJobConfiguration.setAmQueue("default");
- sparkJobConfiguration.setAmMemory(1024);
- sparkJobConfiguration.setAmVCores(1);
- sparkJobConfiguration.setAppPath("hdfs://" + featurestoreExampleJarDst);
- sparkJobConfiguration.setMainClass(Settings.HOPS_FEATURESTORE_TOUR_JOB_CLASS);
- sparkJobConfiguration.setDefaultArgs("--input TestJob/data");
- sparkJobConfiguration.setExecutorInstances(1);
- sparkJobConfiguration.setExecutorCores(1);
- sparkJobConfiguration.setExecutorMemory(2024);
- sparkJobConfiguration.setExecutorGpus(0);
- sparkJobConfiguration.setDynamicAllocationEnabled(true);
- sparkJobConfiguration.setDynamicAllocationMinExecutors(1);
- sparkJobConfiguration.setDynamicAllocationMaxExecutors(3);
- sparkJobConfiguration.setDynamicAllocationInitialExecutors(1);
- sparkJobConfiguration.setAppName(Settings.HOPS_FEATURESTORE_TOUR_JOB_NAME);
- sparkJobConfiguration.setLocalResources(new LocalResourceDTO[0]);
- Jobs job = jobController.putJob(user, project, null, sparkJobConfiguration);
- activityFacade.persistActivity(ActivityFacade.CREATED_JOB + job.getName(), project, user,
- ActivityFlag.SERVICE);
- executionController
- .start(job, Settings.HOPS_FEATURESTORE_TOUR_JOB_INPUT_PARAM + tourFilesDataset + "/data", user);
- activityFacade.persistActivity(ActivityFacade.RAN_JOB + job.getName(), project, user,
- ActivityFlag.SERVICE);
  break;
  default:
  break;

diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java
@@ -2470,11 +2470,6 @@ public synchronized String getHopsExamplesSparkFilename() {
  return "hops-examples-spark-" + HOPS_EXAMPLES_VERSION + ".jar";
  }
 
- public synchronized String getHopsExamplesFeaturestoreTourFilename() {
- checkCache();
- return "hops-examples-featurestore-tour-" + HOPS_EXAMPLES_VERSION + ".jar";
- }
-
  private String VERIFICATION_PATH = "/hopsworks-admin/security/validate_account.xhtml";
 
  public synchronized String getEmailVerificationEndpoint() {