[flang] Implement reading of module files

When a use-stmt is encountered for a module that isn't in the global
scope, search for and read the appropriate `.mod` file. To perform the
search, pass the search directories in to ResolveNames.

For modules that were read from `.mod` files, we have to keep the cooked
source from being deleted so that the names so that references to names
stay valid. So we store the cooked source in the Scope of the module as
a `unique_ptr`.

Add `Symbol::Flag::ModFile` to distinguish module symbols that were read
from a `.mod` file rather than from the current compilation. Use it to
prevent writing those back out.

Fix test_errors.sh to run the compiler in the temp subdirectory --
otherwise tests could be affected by `.mod` files left from previous
tests.

Original-commit: flang-compiler/f18@207065999c
Reviewed-on: https://github.com/flang-compiler/f18/pull/145

flang/documentation/mod-files.md

@@ -102,6 +102,11 @@ For PGI, `-I` specifies directories to search for include files and module
 files. `-module` specifics a directory to write module files in as well as to
 search for them. gfortran is similar except it uses `-J` instead of `-module`.
 
+The search order for module files is:
+1. The `-module` directory (Note: for gfortran the `-J` directory is not searched).
+2. The current directory
+3. The `-I` directories in the order they appear on the command line
+
 ### Writing module files
 
 When writing a module file, if the existing one matches what would be written,

flang/lib/semantics/mod-file.cc

@@ -15,7 +15,11 @@
 #include "mod-file.h"
 #include "scope.h"
 #include "symbol.h"
+#include "../parser/grammar.h"
 #include "../parser/message.h"
+#include "../parser/openmp-grammar.h"
+#include "../parser/preprocessor.h"
+#include "../parser/prescan.h"
 #include <algorithm>
 #include <cerrno>
 #include <fstream>
@@ -23,18 +27,28 @@
 #include <ostream>
 #include <set>
 #include <sstream>
+#include <sys/stat.h>
+#include <sys/types.h>
 #include <vector>
 
 namespace Fortran::semantics {
 
 using namespace parser::literals;
 
-class ModFileWriter {
-public:
+// The extension used for module files.
+static constexpr auto extension{".mod"};
 // The initial characters of a file that identify it as a .mod file.
 static constexpr auto magic{"!mod$"};
-  static constexpr auto extension{".mod"};
+// Construct the path to a module file.
+static std::string ModFilePath(const std::string &, const std::string &);
+// Helpers for creating error messages.
+static parser::Message Error(
+    const SourceName &, parser::MessageFixedText, const std::string &);
+static parser::Message Error(const SourceName &, parser::MessageFixedText,
+    const std::string &, const std::string &);
 
+class ModFileWriter {
+public:
   // The .mod file format version number.
   void set_version(int version) { version_ = version; }
   // The directory to write .mod files in.
@@ -103,16 +117,18 @@ bool ModFileWriter::WriteAll() {
   for (const auto &scope : Scope::globalScope.children()) {
     if (scope.kind() == Scope::Kind::Module) {
       auto &symbol{*scope.symbol()};  // symbol must be present for module
+      if (!symbol.test(Symbol::Flag::ModFile)) {
         WriteOne(symbol);
       }
     }
+  }
   return errors_.empty();
 }
 
 bool ModFileWriter::WriteOne(const Symbol &modSymbol) {
   CHECK(modSymbol.has<ModuleDetails>());
   auto name{parser::ToLowerCaseLetters(modSymbol.name().ToString())};
-  std::string path{dir_ + '/' + name + extension};
+  std::string path{ModFilePath(dir_, name)};
   std::ofstream os{path};
   PutSymbols(*modSymbol.scope());
   std::string all{GetAsString(name)};
@@ -396,4 +412,103 @@ std::string ModFileWriter::CheckSum(const std::string &str) {
 
 void WriteModFiles() { ModFileWriter{}.WriteAll(std::cerr); }
 
+bool ModFileReader::Read(const SourceName &modName) {
+  auto path{FindModFile(modName)};
+  if (!path.has_value()) {
+    return false;
+  }
+  if (!Prescan(modName, *path)) {
+    return false;
+  }
+  parser::ParseState parseState{*cooked_};
+  auto parseTree{parser::program.Parse(parseState)};
+  if (!parseState.messages().empty()) {
+    errors_.emplace_back(modName,
+        parser::MessageFormattedText{
+            "Module file for '%s' is corrupt: %s"_err_en_US,
+            modName.ToString().data(), path->data()});
+    return false;
+  }
+  ResolveNames(*parseTree, *cooked_, directories_);
+  const auto &it{Scope::globalScope.find(modName)};
+  if (it == Scope::globalScope.end()) {
+    return false;
+  }
+  auto &modSymbol{*it->second};
+  modSymbol.scope()->set_cookedSource(std::move(cooked_));
+  modSymbol.set(Symbol::Flag::ModFile);
+  return true;
+}
+
+// Look for the .mod file for this module in the search directories.
+// Add to errors_ if not found.
+std::optional<std::string> ModFileReader::FindModFile(
+    const SourceName &modName) {
+  auto error{Error(modName, "Cannot find module file for '%s'"_err_en_US,
+      modName.ToString())};
+  for (auto &dir : directories_) {
+    std::string path{ModFilePath(dir, modName.ToString())};
+    std::ifstream ifstream{path};
+    if (!ifstream.good()) {
+      error.Attach(Error(
+          modName, "%s: %s"_en_US, path, std::string{std::strerror(errno)}));
+    } else {
+      std::string line;
+      ifstream >> line;
+      if (std::equal(line.begin(), line.end(), std::string{magic}.begin())) {
+        return path;  // success
+      }
+      error.Attach(Error(modName, "%s: Not a valid module file"_en_US, path));
+    }
+  }
+  errors_.push_back(error);
+  return std::nullopt;
+}
+
+bool ModFileReader::Prescan(
+    const SourceName &modName, const std::string &path) {
+  std::stringstream fileError;
+  const auto *sourceFile{allSources_.Open(path, &fileError)};
+  if (sourceFile == nullptr) {
+    errors_.push_back(
+        Error(modName, "Cannot read %s: %s"_err_en_US, path, fileError.str()));
+    return false;
+  }
+  parser::Preprocessor preprocessor{allSources_};
+  parser::Messages messages;
+  parser::Prescanner prescanner{messages, *cooked_, preprocessor, {}};
+  parser::ProvenanceRange range{
+      allSources_.AddIncludedFile(*sourceFile, parser::ProvenanceRange{})};
+  prescanner.Prescan(range);
+  if (!messages.empty()) {
+    errors_.push_back(
+        Error(modName, "Module file for '%s' is corrupt: %s"_err_en_US,
+            modName.ToString(), path));
+    return false;
+  }
+  cooked_->Marshal();
+  return true;
+}
+
+static std::string ModFilePath(
+    const std::string &dir, const std::string &modName) {
+  if (dir == "."s) {
+    return modName + extension;
+  } else {
+    return dir + '/' + modName + extension;
+  }
+}
+
+static parser::Message Error(const SourceName &location,
+    parser::MessageFixedText fixedText, const std::string &arg) {
+  return parser::Message{
+      location, parser::MessageFormattedText{fixedText, arg.data()}};
+}
+static parser::Message Error(const SourceName &location,
+    parser::MessageFixedText fixedText, const std::string &arg1,
+    const std::string &arg2) {
+  return parser::Message{location,
+      parser::MessageFormattedText{fixedText, arg1.data(), arg2.data()}};
+}
+
 }  // namespace Fortran::semantics

flang/lib/semantics/mod-file.h

@@ -15,10 +15,43 @@
 #ifndef FORTRAN_SEMANTICS_MOD_FILE_H_
 #define FORTRAN_SEMANTICS_MOD_FILE_H_
 
+#include "resolve-names.h"
+#include "../parser/char-block.h"
+#include "../parser/message.h"
+#include "../parser/parse-tree.h"
+#include "../parser/parsing.h"
+#include "../parser/provenance.h"
+#include <iostream>
+#include <string>
+
 namespace Fortran::semantics {
 
+using SourceName = parser::CharBlock;
+
 void WriteModFiles();
 
+class ModFileReader {
+public:
+  // directories specifies where to search for module files
+  ModFileReader(const std::vector<std::string> &directories)
+    : directories_{directories} {}
+
+  // Find and read the module file for modName.
+  // Return true on success; otherwise errors() reports the problems.
+  bool Read(const SourceName &modName);
+  std::list<parser::Message> &errors() { return errors_; }
+
+private:
+  std::vector<std::string> directories_;
+  parser::AllSources allSources_;
+  std::unique_ptr<parser::CookedSource> cooked_{
+      std::make_unique<parser::CookedSource>(allSources_)};
+  std::list<parser::Message> errors_;
+
+  std::optional<std::string> FindModFile(const SourceName &);
+  bool Prescan(const SourceName &, const std::string &);
+};
+
 }  // namespace Fortran::semantics
 
 #endif

flang/lib/semantics/resolve-names.cc

@@ -27,6 +27,7 @@
 #include <ostream>
 #include <set>
 #include <stack>
+#include <vector>
 
 namespace Fortran::semantics {
 
@@ -387,6 +388,10 @@ public:
   bool Pre(const parser::UseStmt &);
   void Post(const parser::UseStmt &);
 
+  void add_searchDirectory(const std::string &dir) {
+    searchDirectories_.push_back(dir);
+  }
+
 private:
   // The default access spec for this module.
   Attr defaultAccess_{Attr::PUBLIC};
@@ -394,9 +399,12 @@ private:
   const SourceName *prevAccessStmt_{nullptr};
   // The scope of the module during a UseStmt
   const Scope *useModuleScope_{nullptr};
+  // Directories to search for .mod files
+  std::vector<std::string> searchDirectories_;
+
   void SetAccess(const parser::Name &, Attr);
   void ApplyDefaultAccess();
-
+  const Scope *FindModule(const SourceName &);
   void AddUse(const parser::Rename::Names &);
   void AddUse(const parser::Name &);
   // Record a use from useModuleScope_ of useName as localName. location is
@@ -1223,20 +1231,8 @@ bool ModuleVisitor::Pre(const parser::Rename::Names &x) {
 
 // Set useModuleScope_ to the Scope of the module being used.
 bool ModuleVisitor::Pre(const parser::UseStmt &x) {
-  // x.nature = UseStmt::ModuleNature::Intrinsic or Non_Intrinsic
-  const auto it{Scope::globalScope.find(x.moduleName.source)};
-  if (it == Scope::globalScope.end()) {
-    Say(x.moduleName, "Module '%s' not found"_err_en_US);
-    return false;
-  }
-  const auto *details{it->second->detailsIf<ModuleDetails>()};
-  if (!details) {
-    Say(x.moduleName, "'%s' is not a module"_err_en_US);
-    return false;
-  }
-  useModuleScope_ = details->scope();
-  CHECK(useModuleScope_);
-  return true;
+  useModuleScope_ = FindModule(x.moduleName.source);
+  return useModuleScope_ != nullptr;
 }
 void ModuleVisitor::Post(const parser::UseStmt &x) {
   if (const auto *list{std::get_if<std::list<parser::Rename>>(&x.u)}) {
@@ -1270,6 +1266,30 @@ void ModuleVisitor::Post(const parser::UseStmt &x) {
   useModuleScope_ = nullptr;
 }
 
+// Find the module with this name and return its scope.
+// May have to read a .mod file to find it.
+// Return nullptr on error, after reporting it.
+const Scope *ModuleVisitor::FindModule(const SourceName &name) {
+  auto it{Scope::globalScope.find(name)};
+  if (it == Scope::globalScope.end()) {
+    ModFileReader reader{searchDirectories_};
+    if (!reader.Read(name)) {
+      for (auto &error : reader.errors()) {
+        Say(std::move(error));
+      }
+      return nullptr;
+    }
+    it = Scope::globalScope.find(name);
+    CHECK(it != Scope::globalScope.end());  // else would have reported error
+  }
+  const auto *details{it->second->detailsIf<ModuleDetails>()};
+  if (!details) {
+    Say(name, "'%s' is not a module"_err_en_US);
+    return nullptr;
+  }
+  return details->scope();
+}
+
 void ModuleVisitor::AddUse(const parser::Rename::Names &names) {
   const SourceName &useName{std::get<0>(names.t).source};
   const SourceName &localName{std::get<1>(names.t).source};
@@ -2421,9 +2441,13 @@ void ResolveNamesVisitor::Post(const parser::Program &) {
   CHECK(!GetDeclTypeSpec());
 }
 
-void ResolveNames(
-    parser::Program &program, const parser::CookedSource &cookedSource) {
+void ResolveNames(parser::Program &program,
+    const parser::CookedSource &cookedSource,
+    const std::vector<std::string> &searchDirectories) {
   ResolveNamesVisitor visitor;
+  for (auto &dir : searchDirectories) {
+    visitor.add_searchDirectory(dir);
+  }
   parser::Walk(const_cast<const parser::Program &>(program), visitor);
   if (!visitor.messages().empty()) {
     visitor.messages().Emit(std::cerr, cookedSource);

flang/lib/semantics/resolve-names.h

@@ -16,6 +16,7 @@
 #define FORTRAN_SEMANTICS_RESOLVE_NAMES_H_
 
 #include <iosfwd>
+#include <vector>
 
 namespace Fortran::parser {
 struct Program;
@@ -24,7 +25,8 @@ class CookedSource;
 
 namespace Fortran::semantics {
 
-void ResolveNames(parser::Program &, const parser::CookedSource &);
+void ResolveNames(parser::Program &, const parser::CookedSource &,
+    const std::vector<std::string> &);
 void DumpSymbols(std::ostream &);
 
 }  // namespace Fortran::semantics

flang/lib/semantics/scope.cc

@@ -60,8 +60,8 @@ std::ostream &operator<<(std::ostream &os, const Scope &scope) {
   }
   os << scope.children_.size() << " children\n";
   for (const auto &pair : scope.symbols_) {
-    const auto &symbol{pair.second};
-    os << "  " << symbol << '\n';
+    const auto *symbol{pair.second};
+    os << "  " << *symbol << '\n';
   }
   return os;
 }

flang/lib/semantics/scope.h

@@ -106,6 +106,11 @@ public:
 
   DerivedTypeSpec &MakeDerivedTypeSpec(const SourceName &);
 
+  std::unique_ptr<parser::CookedSource> cooked_;
+  void set_cookedSource(std::unique_ptr<parser::CookedSource> cooked) {
+    cooked_ = std::move(cooked);
+  }
+
 private:
   Scope &parent_;
   const Kind kind_;

flang/lib/semantics/symbol.h

@@ -233,7 +233,7 @@ std::string DetailsToString(const Details &);
 
 class Symbol {
 public:
-  ENUM_CLASS(Flag, Function, Subroutine, Implicit);
+  ENUM_CLASS(Flag, Function, Subroutine, Implicit, ModFile);
   using Flags = common::EnumSet<Flag, Flag_enumSize>;
 
   const Scope &owner() const { return *owner_; }

flang/test/semantics/resolve12.f90

@@ -19,7 +19,7 @@ subroutine sub
 end
 
 use m1
-!ERROR: Module 'm2' not found
+!ERROR: Cannot find module file for 'm2'
 use m2
 !ERROR: 'sub' is not a module
 use sub

flang/test/semantics/test_errors.sh

@@ -18,7 +18,7 @@
 
 PATH=/usr/bin:/bin
 srcdir=$(dirname $0)
-CMD="${F18:-../../tools/f18/f18} -fdebug-resolve-names -fparse-only"
+CMD="${F18:-../../../tools/f18/f18} -fdebug-resolve-names -fparse-only"
 
 if [[ $# != 1 ]]; then
   echo "Usage: $0 <fortran-source>"
@@ -38,7 +38,7 @@ expect=$temp/expect
 diffs=$temp/diffs
 
 cmd="$CMD $src"
-$cmd > $log 2>&1
+( cd $temp; $cmd ) > $log 2>&1
 [[ $? -ge 128 ]] && exit 1
 
 # $actual has errors from the compiler; $expect has them from !ERROR comments in source

flang/tools/f18/f18.cc

@@ -206,7 +206,9 @@ std::string CompileFortran(
   }
   if (driver.debugResolveNames || driver.dumpSymbols ||
       driver.dumpUnparseWithSymbols) {
-    Fortran::semantics::ResolveNames(parseTree, parsing.cooked());
+    std::vector<std::string> directories{options.searchDirectories};
+    directories.insert(directories.begin(), "."s);
+    Fortran::semantics::ResolveNames(parseTree, parsing.cooked(), directories);
     Fortran::semantics::WriteModFiles();
     if (driver.dumpSymbols) {
       Fortran::semantics::DumpSymbols(std::cout);
@@ -452,7 +454,8 @@ int main(int argc, char *const argv[]) {
   if (options.isStrictlyStandard) {
     options.features.WarnOnAllNonstandard();
   }
-  if (!options.features.IsEnabled(Fortran::parser::LanguageFeature::BackslashEscapes)) {
+  if (!options.features.IsEnabled(
+          Fortran::parser::LanguageFeature::BackslashEscapes)) {
     driver.pgf90Args.push_back("-Mbackslash");
   }