Relative paths are relative to current project locally and in Cloud (#10660)

- Close #10622
- Changes `project-manager` and `ensoup` launcher to run the engine/language-server with working directory set to the directory containing currently running project.
- If the working directory is _not_ "the directory containing currently running project", a warning is written to logs. This can happen if the raw `/bin/enso` engine runner is used in a different directory.
- In the Cloud, the `File.new` interprets relative paths as cloud paths relative to the Cloud directory containing the current project. Absolute paths are unaffected.

CHANGELOG.md

@@ -4,8 +4,11 @@
 
 - [Implemented in-memory and database mixed `Decimal` column
   comparisons.][10614]
+- [Relative paths are now resolved relative to the project location, also in the
+  Cloud.][10660]
 
 [10614]: https://github.com/enso-org/enso/pull/10614
+[10660]: https://github.com/enso-org/enso/pull/10660
 
 # Enso 2023.3
 

build.sbt

@@ -3860,7 +3860,6 @@ pkgStdLibInternal := Def.inputTask {
     if (generateIndex) {
       val stdlibStandardRoot = root / "lib" / standardNamespace
       DistributionPackage.indexStdLib(
-        libMajor       = stdlibStandardRoot,
         libName        = stdlibStandardRoot / lib,
         stdLibVersion  = defaultDevEnsoVersion,
         ensoVersion    = defaultDevEnsoVersion,

build/build/src/enso.rs

@@ -78,6 +78,7 @@ impl BuiltEnso {
 
     pub async fn run_benchmarks(&self, opt: BenchmarkOptions) -> Result {
         let filename = format!("enso{}", if TARGET_OS == OS::Windows { ".exe" } else { "" });
+        let base_working_directory = self.paths.repo_root.test.benchmarks.try_parent()?;
         let enso = self
             .paths
             .repo_root
@@ -88,6 +89,7 @@ impl BuiltEnso {
             .join(filename);
         let benchmarks = Command::new(&enso)
             .args(["--jvm", "--run", self.paths.repo_root.test.benchmarks.as_str()])
+            .current_dir(base_working_directory)
             .set_env(ENSO_BENCHMARK_TEST_DRY_RUN, &Boolean::from(opt.dry_run))?
             .run_ok()
             .await;
@@ -96,10 +98,12 @@ impl BuiltEnso {
 
     pub fn run_test(&self, test_path: impl AsRef<Path>, ir_caches: IrCaches) -> Result<Command> {
         let mut command = self.cmd()?;
+        let base_working_directory = test_path.try_parent()?;
         command
             .arg(ir_caches)
             .arg("--run")
             .arg(test_path.as_ref())
+            .current_dir(base_working_directory)
             // This flag enables assertions in the JVM. Some of our stdlib tests had in the past
             // failed on Graal/Truffle assertions, so we want to have them triggered.
             .set_env(JAVA_OPTS, &ide_ci::programs::java::Option::EnableAssertions.as_ref())?;

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Enso_File.enso

@@ -88,9 +88,15 @@ type Enso_File
        directory.
     current_working_directory : Enso_File
     current_working_directory =
+        Enso_File.cloud_project_parent_directory . if_nothing Enso_File.root
+
+    ## PRIVATE
+       The parent directory containing the currently open project if in the
+       Cloud, or `Nothing` if running locally.
+    cloud_project_parent_directory : Enso_File | Nothing
+    cloud_project_parent_directory =
         path = Environment.get "ENSO_CLOUD_PROJECT_DIRECTORY_PATH"
-        if path.is_nothing then Enso_File.root else
-            Enso_File.new path
+        path.if_not_nothing <| Enso_File.new path
 
     ## PRIVATE
     asset_type self -> Enso_Asset_Type =

distribution/lib/Standard/Base/0.0.0-dev/src/System/File.enso

@@ -10,6 +10,7 @@ import project.Data.Time.Date_Time.Date_Time
 import project.Data.Vector.Vector
 import project.Enso_Cloud.Data_Link.Data_Link
 import project.Enso_Cloud.Data_Link_Helpers
+import project.Enso_Cloud.Enso_File.Enso_File
 import project.Error.Error
 import project.Errors.Common.Dry_Run_Operation
 import project.Errors.Common.Type_Error
@@ -61,6 +62,10 @@ type File
 
        Creates a new file object, pointing to the given path.
 
+       Relative paths are resolved relative to the directory containing the
+       currently running workflow. Thus, if the workflow is running in the Cloud,
+       the relative paths will be resolved to Cloud files.
+
        Arguments:
        - path: The path to the file that you want to create, or a file itself. The
          latter is a no-op.
@@ -74,7 +79,7 @@ type File
              example_new = File.new Examples.csv_path
     new : (Text | File) -> Any ! Illegal_Argument
     new path = case path of
-        _ : Text -> if path.contains "://" . not then get_file path else
+        _ : Text -> if path.contains "://" . not then resolve_path path else
             protocol = path.split "://" . first
             file_system = FileSystemSPI.get_type protocol False
             if file_system.is_nothing then Error.throw (Illegal_Argument.Error "Unsupported protocol "+protocol) else
@@ -860,6 +865,22 @@ get_cwd = @Builtin_Method "File.get_cwd"
 get_file : Text -> File
 get_file path = @Builtin_Method "File.get_file"
 
+## PRIVATE
+   Resolves the given path to a corresponding file location.
+
+   If the provided path is relative, the behaviour depends on the context:
+   - if the project is running in the Cloud, the path is resolved to a Cloud file,
+     relative to the project's location.
+   - if running locally, the path is resolved to a local file, relative to the
+     current working directory.
+resolve_path (path : Text) -> File | Enso_File =
+    local_file = get_file path
+    # Absolute files always resolve to themselves.
+    if local_file.is_absolute then local_file else
+        case Enso_File.cloud_project_parent_directory of
+            Nothing -> local_file
+            base_cloud_directory -> base_cloud_directory / path
+
 ## PRIVATE
 get_child_widget : File -> Widget
 get_child_widget file =

engine/launcher/src/main/scala/org/enso/launcher/components/LauncherRunner.scala

@@ -55,7 +55,8 @@ class LauncherRunner(
           projectManager.findProject(currentWorkingDirectory).get
       }
 
-      val version = resolveVersion(versionOverride, inProject)
+      val version          = resolveVersion(versionOverride, inProject)
+      val workingDirectory = workingDirectoryForRunner(inProject, None)
       val arguments = inProject match {
         case Some(project) =>
           val projectPackagePath =
@@ -68,6 +69,7 @@ class LauncherRunner(
         version,
         arguments ++ setLogLevelArgs(logLevel, logMasking)
         ++ additionalArguments,
+        workingDirectory         = workingDirectory,
         connectLoggerIfAvailable = true
       )
     }
@@ -109,6 +111,10 @@ class LauncherRunner(
         else projectManager.findProject(actualPath).get
       val version = resolveVersion(versionOverride, project)
 
+      // The engine is started in the directory containing the project, or the standalone script.
+      val workingDirectory =
+        workingDirectoryForRunner(project, Some(actualPath))
+
       val arguments =
         if (projectMode) Seq("--run", actualPath.toString)
         else
@@ -127,10 +133,24 @@ class LauncherRunner(
         version,
         arguments ++ setLogLevelArgs(logLevel, logMasking)
         ++ additionalArguments,
+        workingDirectory         = workingDirectory,
         connectLoggerIfAvailable = true
       )
     }
 
+  private def workingDirectoryForRunner(
+    inProject: Option[Project],
+    scriptPath: Option[Path]
+  ): Option[Path] = {
+    // The path of the project or standalone script that is being run.
+    val baseDirectory = inProject match {
+      case Some(project) => Some(project.path)
+      case None          => scriptPath
+    }
+
+    baseDirectory.map(p => p.toAbsolutePath.normalize().getParent)
+  }
+
   private def setLogLevelArgs(
     level: Level,
     logMasking: Boolean
@@ -190,7 +210,12 @@ class LauncherRunner(
         }
 
       (
-        RunSettings(version, arguments, connectLoggerIfAvailable = false),
+        RunSettings(
+          version,
+          arguments,
+          workingDirectory         = None,
+          connectLoggerIfAvailable = false
+        ),
         whichEngine
       )
     }
@@ -239,6 +264,7 @@ class LauncherRunner(
         version,
         arguments ++ setLogLevelArgs(logLevel, logMasking)
         ++ additionalArguments,
+        workingDirectory         = None,
         connectLoggerIfAvailable = true
       )
     }
@@ -286,6 +312,7 @@ class LauncherRunner(
         version,
         arguments ++ setLogLevelArgs(logLevel, logMasking)
         ++ additionalArguments,
+        workingDirectory         = None,
         connectLoggerIfAvailable = true
       )
     }

engine/launcher/src/test/scala/org/enso/launcher/components/LauncherRunnerSpec.scala

@@ -64,6 +64,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
       val runSettings = RunSettings(
         SemVer.of(0, 0, 0),
         Seq("arg1", "--flag2"),
+        workingDirectory         = None,
         connectLoggerIfAvailable = true
       )
       val jvmOptions = Seq(("locally-added-options", "value1"))
@@ -243,6 +244,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
         .get
 
       outsideProject.engineVersion shouldEqual version
+      outsideProject.workingDirectory shouldEqual Some(projectPath.getParent)
       outsideProject.runnerArguments.mkString(" ") should
       (include(s"--in-project $normalizedPath") and include("--repl"))
 
@@ -258,6 +260,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
         .get
 
       insideProject.engineVersion shouldEqual version
+      insideProject.workingDirectory shouldEqual Some(projectPath.getParent)
       insideProject.runnerArguments.mkString(" ") should
       (include(s"--in-project $normalizedPath") and include("--repl"))
 
@@ -304,6 +307,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
         .get
 
       runSettings.engineVersion shouldEqual version
+      runSettings.workingDirectory shouldEqual Some(projectPath.getParent)
       val commandLine = runSettings.runnerArguments.mkString(" ")
       commandLine should include(s"--interface ${options.interface}")
       commandLine should include(s"--rpc-port ${options.rpcPort}")
@@ -346,6 +350,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
         .get
 
       outsideProject.engineVersion shouldEqual version
+      outsideProject.workingDirectory shouldEqual Some(projectPath.getParent)
       outsideProject.runnerArguments.mkString(" ") should
       include(s"--run $normalizedPath")
 
@@ -443,6 +448,7 @@ class LauncherRunnerSpec extends RuntimeVersionManagerTest with FlakySpec {
         .get
 
       runSettings.engineVersion shouldEqual version
+      runSettings.workingDirectory shouldEqual Some(projectPath.getParent)
       runSettings.runnerArguments.mkString(" ") should
       (include(s"--run $normalizedFilePath") and
       include(s"--in-project $normalizedProjectPath"))

engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java

@@ -21,10 +21,12 @@ import com.oracle.truffle.api.object.Shape;
 import com.oracle.truffle.api.source.Source;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.PrintStream;
 import java.net.MalformedURLException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
@@ -56,6 +58,7 @@ import org.enso.interpreter.runtime.state.State;
 import org.enso.interpreter.runtime.util.TruffleFileSystem;
 import org.enso.librarymanager.ProjectLoadingFailure;
 import org.enso.librarymanager.resolved.LibraryRoot;
+import org.enso.logger.masking.MaskedPath$;
 import org.enso.pkg.Package;
 import org.enso.pkg.PackageManager;
 import org.enso.pkg.QualifiedName;
@@ -164,6 +167,7 @@ public final class EnsoContext {
     PackageManager<TruffleFile> packageManager = new PackageManager<>(fs);
 
     Optional<TruffleFile> projectRoot = OptionsHelper.getProjectRoot(environment);
+    checkWorkingDirectory(projectRoot);
     Optional<Package<TruffleFile>> projectPackage =
         projectRoot.map(
             file ->
@@ -207,6 +211,28 @@ public final class EnsoContext {
     }
   }
 
+  /** Checks if the working directory is as expected and reports a warning if not. */
+  private void checkWorkingDirectory(Optional<TruffleFile> maybeProjectRoot) {
+    if (maybeProjectRoot.isPresent()) {
+      var root = maybeProjectRoot.get();
+      var parent = root.getAbsoluteFile().normalize().getParent();
+      var cwd = environment.getCurrentWorkingDirectory().getAbsoluteFile().normalize();
+      try {
+        if (!cwd.isSameFile(parent)) {
+          var maskedPath = MaskedPath$.MODULE$.apply(Path.of(parent.toString()));
+          logger.log(
+              Level.WARNING,
+              "Initializing the context in a different working directory than the one containing"
+                  + " the project root. This may lead to relative paths not behaving as advertised"
+                  + " by `File.new`. Please run the engine inside of `{}` directory.",
+              maskedPath);
+        }
+      } catch (IOException e) {
+        logger.severe("Error checking working directory: " + e.getMessage());
+      }
+    }
+  }
+
   /**
    * @param node the location of context access. Pass {@code null} if not in a node.
    * @return the proper context instance for the current {@link

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Command.scala

@@ -3,6 +3,7 @@ package org.enso.runtimeversionmanager.runner
 import com.typesafe.scalalogging.Logger
 import org.enso.process.WrappedProcess
 
+import java.nio.file.Path
 import scala.sys.process.Process
 import scala.util.{Failure, Try}
 
@@ -10,8 +11,13 @@ import scala.util.{Failure, Try}
   *
   * @param command the command and its arguments that should be executed
   * @param extraEnv environment variables that should be overridden
+  * @param workingDirectory the working directory in which the command should be executed (if None, the working directory is not overridden and is inherited instead)
   */
-case class Command(command: Seq[String], extraEnv: Seq[(String, String)]) {
+case class Command(
+  command: Seq[String],
+  extraEnv: Seq[(String, String)],
+  workingDirectory: Option[Path]
+) {
   private val logger = Logger[Command]
 
   /** Runs the command and returns its exit code.
@@ -79,6 +85,7 @@ case class Command(command: Seq[String], extraEnv: Seq[(String, String)]) {
     for ((key, value) <- extraEnv) {
       processBuilder.environment().put(key, value)
     }
+    workingDirectory.foreach(path => processBuilder.directory(path.toFile))
     processBuilder
   }
 

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/RunSettings.scala

@@ -2,15 +2,19 @@ package org.enso.runtimeversionmanager.runner
 
 import org.enso.semver.SemVer
 
+import java.nio.file.Path
+
 /** Represents settings that are used to launch the runner JAR.
   *
   * @param engineVersion Enso engine version to use
   * @param runnerArguments arguments that should be passed to the runner
+  * @param workingDirectory the working directory override
   * @param connectLoggerIfAvailable specifies if the ran component should
   *                                 connect to launcher's logging service
   */
 case class RunSettings(
   engineVersion: SemVer,
   runnerArguments: Seq[String],
+  workingDirectory: Option[Path],
   connectLoggerIfAvailable: Boolean
 )

lib/scala/runtime-version-manager/src/main/scala/org/enso/runtimeversionmanager/runner/Runner.scala

@@ -79,7 +79,12 @@ class Runner(
           engineVersion
         )
       }
-      RunSettings(engineVersion, arguments, connectLoggerIfAvailable = false)
+      RunSettings(
+        engineVersion,
+        arguments,
+        workingDirectory         = None,
+        connectLoggerIfAvailable = false
+      )
     }
 
   /** Creates [[RunSettings]] for launching the Language Server. */
@@ -113,6 +118,8 @@ class Runner(
     additionalArguments: Seq[String]
   ): Try[RunSettings] =
     Try {
+      val workingDirectory =
+        Path.of(projectPath).toAbsolutePath.normalize.getParent
       val arguments = Seq(
         "--server",
         "--root-id",
@@ -137,6 +144,7 @@ class Runner(
       RunSettings(
         version,
         arguments ++ additionalArguments,
+        workingDirectory         = Some(workingDirectory),
         connectLoggerIfAvailable = true
       )
     }
@@ -238,7 +246,13 @@ class Runner(
       val extraEnvironmentOverrides =
         javaHome.map("JAVA_HOME" -> _).toSeq ++ distributionSettings.toSeq
 
-      action(Command(command, extraEnvironmentOverrides))
+      action(
+        Command(
+          command,
+          extraEnvironmentOverrides,
+          runSettings.workingDirectory
+        )
+      )
     }
 
     val engineVersion = runSettings.engineVersion

project/DistributionPackage.scala

@@ -210,7 +210,6 @@ object DistributionPackage {
       libName  <- (stdLibRoot / libMajor.getName).listFiles()
     } yield {
       indexStdLib(
-        libMajor,
         libName,
         stdLibVersion,
         ensoVersion,
@@ -222,7 +221,6 @@ object DistributionPackage {
   }
 
   def indexStdLib(
-    libMajor: File,
     libName: File,
     stdLibVersion: String,
     ensoVersion: String,
@@ -239,25 +237,25 @@ object DistributionPackage {
       path.globRecursive("*.enso" && FileOnlyFilter).get().toSet
     ) { diff =>
       if (diff.modified.nonEmpty) {
-        log.info(s"Generating index for ${libName} ")
+        log.info(s"Generating index for $libName ")
         val command = Seq(
-          Platform.executableFileName(ensoExecutable.toString),
+          Platform.executableFile(ensoExecutable.getAbsoluteFile),
           "--no-compile-dependencies",
           "--no-global-cache",
           "--compile",
-          path.toString
+          path.getAbsolutePath
         )
         log.debug(command.mkString(" "))
         val exitCode = Process(
           command,
-          None,
+          Some(path.getAbsoluteFile.getParentFile),
           "JAVA_OPTS" -> "-Dorg.jline.terminal.dumb=true"
         ).!
         if (exitCode != 0) {
-          throw new RuntimeException(s"Cannot compile $libMajor.$libName.")
+          throw new RuntimeException(s"Cannot compile $libName.")
         }
       } else {
-        log.debug(s"No modified files. Not generating index for ${libName}.")
+        log.debug(s"No modified files. Not generating index for $libName.")
       }
     }
   }

project/LibraryManifestGenerator.scala

@@ -53,6 +53,7 @@ object LibraryManifestGenerator {
     javaOpts: Seq[String],
     log: Logger
   ): Unit = {
+    val canonicalPath = projectPath.getCanonicalFile
     val javaCommand =
       ProcessHandle.current().info().command().asScala.getOrElse("java")
     val command = Seq(
@@ -60,7 +61,7 @@ object LibraryManifestGenerator {
     ) ++ javaOpts ++ Seq(
       "--update-manifest",
       "--in-project",
-      projectPath.getCanonicalPath
+      canonicalPath.toString
     )
 
     val commandText = command.mkString(" ")
@@ -68,7 +69,7 @@ object LibraryManifestGenerator {
     val exitCode = sys.process
       .Process(
         command,
-        None,
+        cwd = Some(canonicalPath.getParentFile),
         "ENSO_EDITION_PATH" -> file("distribution/editions").getCanonicalPath
       )
       .!

project/Platform.scala

@@ -1,3 +1,7 @@
+import sbt.singleFileFinder
+
+import java.io.File
+
 object Platform {
 
   /** Returns true if the build system is running on Windows.
@@ -46,4 +50,17 @@ object Platform {
     if (isWindows) s".\\$name.bat" else name
   }
 
+  /** Returns the executable file on the current platform.
+    *
+    * @param file the generic executable path
+    * @return the file corresponding to the provided executable on the current platform
+    */
+  def executableFile(file: File): String =
+    if (isWindows) {
+      val parent = file.getParentFile
+      if (parent == null) s".\\${file.getName}.bat"
+      else if (parent.isAbsolute)
+        new File(parent, s"${file.getName}.bat").getPath
+      else s".\\${parent.getPath}${file.getPath}.bat"
+    } else file.getPath
 }

test/Base_Tests/src/System/File_Spec.enso

@@ -12,6 +12,9 @@ import Standard.Base.System.File.Generic.Writable_File.Writable_File
 polyglot java import org.enso.base_test_helpers.FileSystemHelper
 
 from Standard.Test import all
+import Standard.Test.Test_Environment
+
+import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup
 
 ## We rely on a less strict equality for `File` as it is fine if a relative file gets resolved to absolute.
 File.should_equal self other frames_to_skip=0 =
@@ -361,6 +364,57 @@ add_specs suite_builder =
             (File.new ".").name . should_equal (File.new "." . absolute . normalize . name)
             (File.new "..").name . should_equal (File.new "." . parent . absolute . normalize . name)
 
+        current_project_root = enso_project.root
+        base_directory = current_project_root.parent
+        is_correct_working_directory = (File.current_directory . normalize . path) == current_project_root.absolute.normalize.path
+        group_builder.specify "will resolve relative paths relative to the currently running project" pending=(if is_correct_working_directory.not then "The working directory is not set-up as expected, so this test cannot run. Please run the tests using `ensoup` to ensure the working directory is correct.") <|
+            root = File.new "."
+            root.should_be_a File
+            # The `.` path should resolve to the base path
+            root.absolute.normalize.path . should_equal base_directory.absolute.normalize.path
+
+            expected_file = base_directory / "abc" / "def.txt"
+            f = File.new "abc/def.txt"
+            f.should_be_a File
+            f.absolute.normalize.path . should_equal expected_file.absolute.normalize.path
+
+            (File.new "abc").create_directory . should_succeed
+            txt = "test-content"+Random.uuid
+            txt.write expected_file . should_succeed
+            Panic.with_finalizer expected_file.delete_if_exists <|
+                f.read_text . should_equal txt
+                Data.read "abc/def.txt" . should_equal txt
+
+        cloud_setup = Cloud_Tests_Setup.prepare
+        with_temporary_cloud_root ~action =
+            subdir = (Enso_File.root / ("my_test_CWD-" + Random.uuid.take 5)).create_directory
+            subdir.should_succeed
+            cleanup =
+                Enso_User.flush_caches
+                subdir.delete
+            Panic.with_finalizer cleanup <|
+                Test_Environment.unsafe_with_environment_override "ENSO_CLOUD_PROJECT_DIRECTORY_PATH" subdir.path <|
+                    # Flush caches to ensure fresh dir is used
+                    Enso_User.flush_caches
+                    action
+
+        group_builder.specify "will resolve relative paths as Cloud paths if running in the Cloud" pending=cloud_setup.real_cloud_pending <|
+            with_temporary_cloud_root <|
+                root = File.new "."
+                root.should_be_a Enso_File
+                root.should_equal Enso_File.current_working_directory
+
+                f = File.new "abc/def.txt"
+                f.should_be_a Enso_File
+                f.should_equal (Enso_File.current_working_directory / "abc" / "def.txt")
+
+                # Data.read should be consistent too
+                txt = "test-content"+Random.uuid
+                (File.new "abc").create_directory . should_succeed
+                txt.write f . should_succeed
+                Panic.with_finalizer f.delete_if_exists <|
+                    Data.read "abc/def.txt" . should_equal txt
+
     suite_builder.group "read_text" group_builder->
         group_builder.specify "should allow reading a UTF-8 file" <|
             contents = sample_file.read_text