Auto merge of #62766 - alexcrichton:stabilize-pipelined-compilation, r=oli-obk

rustc: Stabilize options for pipelined compilation

This commit stabilizes options in the compiler necessary for Cargo to
enable "pipelined compilation" by default. The concept of pipelined
compilation, how it's implemented, and what it means for rustc are
documented in #60988. This PR is coupled with a PR against Cargo
(rust-lang/cargo#7143) which updates Cargo's support for pipelined
compliation to rustc, and also enables support by default in Cargo.
(note that the Cargo PR cannot land until this one against rustc lands).

The technical changes performed here were to stabilize the functionality
proposed in #60419 and #60987, the underlying pieces to enable pipelined
compilation support in Cargo. The issues have had some discussion during
stabilization, but the newly stabilized surface area here is:

* A new `--json` flag was added to the compiler.
* The `--json` flag can be passed multiple times.
* The value of the `--json` flag is a comma-separated list of
  directives.
* The `--json` flag cannot be combined with `--color`
* The `--json` flag must be combined with `--error-format=json`
* The acceptable list of directives to `--json` are:
  * `diagnostic-short` - the `rendered` field of diagnostics will have a
    "short" rendering matching `--error-format=short`
  * `diagnostic-rendered-ansi` - the `rendered` field of diagnostics
    will be colorized with ansi color codes embedded in the string field
  * `artifacts` - JSON blobs will be emitted for artifacts being emitted
    by the compiler

The unstable `-Z emit-artifact-notifications` and `--json-rendered`
flags have also been removed during this commit as well.

Closes #60419
Closes #60987
Closes #60988
This commit is contained in:
bors 2019-07-30 08:39:29 +00:00
commit f690098e6d
26 changed files with 237 additions and 112 deletions

View File

@ -229,6 +229,9 @@ impl Step for Cargo {
cargo.env("CFG_DISABLE_CROSS_TESTS", "1");
// Disable a test that has issues with mingw.
cargo.env("CARGO_TEST_DISABLE_GIT_CLI", "1");
// Forcibly disable tests using nightly features since any changes to
// those features won't be able to land.
cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1");
try_run(
builder,

View File

@ -271,3 +271,36 @@ current directory out of pathnames emitted into the object files. The
replacement is purely textual, with no consideration of the current system's
pathname syntax. For example `--remap-path-prefix foo=bar` will match
`foo/lib.rs` but not `./foo/lib.rs`.
## `--json`: configure json messages printed by the compiler
When the `--error-format=json` option is passed to rustc then all of the
compiler's diagnostic output will be emitted in the form of JSON blobs. The
`--json` argument can be used in conjunction with `--error-format=json` to
configure what the JSON blobs contain as well as which ones are emitted.
With `--error-format=json` the compiler will always emit any compiler errors as
a JSON blob, but the following options are also available to the `--json` flag
to customize the output:
- `diagnostic-short` - json blobs for diagnostic messages should use the "short"
rendering instead of the normal "human" default. This means that the output of
`--error-format=short` will be embedded into the JSON diagnostics instead of
the default `--error-format=human`.
- `diagnostic-rendered-ansi` - by default JSON blobs in their `rendered` field
will contain a plain text rendering of the diagnostic. This option instead
indicates that the diagnostic should have embedded ANSI color codes intended
to be used to colorize the message in the manner rustc typically already does
for terminal outputs. Note that this is usefully combined with crates like
`fwdansi` to translate these ANSI codes on Windows to console commands or
`strip-ansi-escapes` if you'd like to optionally remove the ansi colors
afterwards.
- `artifacts` - this instructs rustc to emit a JSON blob for each artifact that
is emitted. An artifact corresponds to a request from the `--emit` CLI
argument, and as soon as the artifact is available on the filesystem a
notification will be emitted.
Note that it is invalid to combine the `--json` argument with the `--color`
argument, and it is required to combine `--json` with `--error-format=json`.

View File

@ -438,6 +438,10 @@ top_level_options!(
remap_path_prefix: Vec<(PathBuf, PathBuf)> [UNTRACKED],
edition: Edition [TRACKED],
// Whether or not we're emitting JSON blobs about each artifact produced
// by the compiler.
json_artifact_notifications: bool [TRACKED],
}
);
@ -625,6 +629,7 @@ impl Default for Options {
cli_forced_thinlto_off: false,
remap_path_prefix: Vec::new(),
edition: DEFAULT_EDITION,
json_artifact_notifications: false,
}
}
}
@ -1459,8 +1464,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
the same values as the target option of the same name"),
allow_features: Option<Vec<String>> = (None, parse_opt_comma_list, [TRACKED],
"only allow the listed language features to be enabled in code (space separated)"),
emit_artifact_notifications: bool = (false, parse_bool, [UNTRACKED],
"emit notifications after each artifact has been output (only in the JSON format)"),
symbol_mangling_version: SymbolManglingVersion = (SymbolManglingVersion::Legacy,
parse_symbol_mangling_version, [TRACKED],
"which mangling version to use for symbol names"),
@ -1818,11 +1821,11 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
"How errors and other messages are produced",
"human|json|short",
),
opt::opt(
opt::multi_s(
"",
"json-rendered",
"Choose `rendered` field of json diagnostics render scheme",
"plain|termcolor",
"json",
"Configure the JSON output of the compiler",
"CONFIG",
),
opt::opt_s(
"",
@ -1918,10 +1921,9 @@ pub fn get_cmd_lint_options(matches: &getopts::Matches,
(lint_opts, describe_lints, lint_cap)
}
pub fn build_session_options_and_crate_config(
matches: &getopts::Matches,
) -> (Options, FxHashSet<(String, Option<String>)>) {
let color = match matches.opt_str("color").as_ref().map(|s| &s[..]) {
/// Parse the `--color` flag
pub fn parse_color(matches: &getopts::Matches) -> ColorConfig {
match matches.opt_str("color").as_ref().map(|s| &s[..]) {
Some("auto") => ColorConfig::Auto,
Some("always") => ColorConfig::Always,
Some("never") => ColorConfig::Never,
@ -1936,46 +1938,52 @@ pub fn build_session_options_and_crate_config(
arg
),
),
};
}
}
let edition = match matches.opt_str("edition") {
Some(arg) => Edition::from_str(&arg).unwrap_or_else(|_|
/// Parse the `--json` flag.
///
/// The first value returned is how to render JSON diagnostics, and the second
/// is whether or not artifact notifications are enabled.
pub fn parse_json(matches: &getopts::Matches) -> (HumanReadableErrorType, bool) {
let mut json_rendered: fn(ColorConfig) -> HumanReadableErrorType =
HumanReadableErrorType::Default;
let mut json_color = ColorConfig::Never;
let mut json_artifact_notifications = false;
for option in matches.opt_strs("json") {
// For now conservatively forbid `--color` with `--json` since `--json`
// won't actually be emitting any colors and anything colorized is
// embedded in a diagnostic message anyway.
if matches.opt_str("color").is_some() {
early_error(
ErrorOutputType::default(),
&format!(
"argument for --edition must be one of: \
{}. (instead was `{}`)",
EDITION_NAME_LIST,
arg
),
),
),
None => DEFAULT_EDITION,
};
"cannot specify the `--color` option with `--json`",
);
}
if !edition.is_stable() && !nightly_options::is_nightly_build() {
early_error(
ErrorOutputType::default(),
&format!(
"Edition {} is unstable and only \
available for nightly builds of rustc.",
edition,
)
)
for sub_option in option.split(',') {
match sub_option {
"diagnostic-short" => json_rendered = HumanReadableErrorType::Short,
"diagnostic-rendered-ansi" => json_color = ColorConfig::Always,
"artifacts" => json_artifact_notifications = true,
s => {
early_error(
ErrorOutputType::default(),
&format!("unknown `--json` option `{}`", s),
)
}
}
}
}
(json_rendered(json_color), json_artifact_notifications)
}
let json_rendered = matches.opt_str("json-rendered").and_then(|s| match s.as_str() {
"plain" => None,
"termcolor" => Some(HumanReadableErrorType::Default(ColorConfig::Always)),
_ => early_error(
ErrorOutputType::default(),
&format!(
"argument for --json-rendered must be `plain` or `termcolor` (instead was `{}`)",
s,
),
),
}).unwrap_or(HumanReadableErrorType::Default(ColorConfig::Never));
/// Parse the `--error-format` flag
pub fn parse_error_format(
matches: &getopts::Matches,
color: ColorConfig,
json_rendered: HumanReadableErrorType,
) -> ErrorOutputType {
// We need the opts_present check because the driver will send us Matches
// with only stable options if no unstable options are used. Since error-format
// is unstable, it will not be present. We have to use opts_present not
@ -2004,6 +2012,60 @@ pub fn build_session_options_and_crate_config(
ErrorOutputType::HumanReadable(HumanReadableErrorType::Default(color))
};
match error_format {
ErrorOutputType::Json { .. } => {}
// Conservatively require that the `--json` argument is coupled with
// `--error-format=json`. This means that `--json` is specified we
// should actually be emitting JSON blobs.
_ if matches.opt_strs("json").len() > 0 => {
early_error(
ErrorOutputType::default(),
"using `--json` requires also using `--error-format=json`",
);
}
_ => {}
}
return error_format;
}
pub fn build_session_options_and_crate_config(
matches: &getopts::Matches,
) -> (Options, FxHashSet<(String, Option<String>)>) {
let color = parse_color(matches);
let edition = match matches.opt_str("edition") {
Some(arg) => Edition::from_str(&arg).unwrap_or_else(|_|
early_error(
ErrorOutputType::default(),
&format!(
"argument for --edition must be one of: \
{}. (instead was `{}`)",
EDITION_NAME_LIST,
arg
),
),
),
None => DEFAULT_EDITION,
};
if !edition.is_stable() && !nightly_options::is_nightly_build() {
early_error(
ErrorOutputType::default(),
&format!(
"Edition {} is unstable and only \
available for nightly builds of rustc.",
edition,
)
)
}
let (json_rendered, json_artifact_notifications) = parse_json(matches);
let error_format = parse_error_format(matches, color, json_rendered);
let unparsed_crate_types = matches.opt_strs("crate-type");
let crate_types = parse_crate_types_from_list(unparsed_crate_types)
.unwrap_or_else(|e| early_error(error_format, &e[..]));
@ -2014,9 +2076,6 @@ pub fn build_session_options_and_crate_config(
let mut debugging_opts = build_debugging_options(matches, error_format);
if !debugging_opts.unstable_options {
if matches.opt_str("json-rendered").is_some() {
early_error(error_format, "`--json-rendered=x` is unstable");
}
if let ErrorOutputType::Json { pretty: true, json_rendered } = error_format {
early_error(
ErrorOutputType::Json { pretty: false, json_rendered },
@ -2441,6 +2500,7 @@ pub fn build_session_options_and_crate_config(
cli_forced_thinlto_off: disable_thinlto,
remap_path_prefix,
edition,
json_artifact_notifications,
},
cfg,
)

View File

@ -95,7 +95,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(sess: &'a Session,
);
}
}
if sess.opts.debugging_opts.emit_artifact_notifications {
if sess.opts.json_artifact_notifications {
sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link");
}
}

View File

@ -696,7 +696,7 @@ fn write_out_deps(compiler: &Compiler, outputs: &OutputFilenames, out_filenames:
match result {
Ok(_) => {
if sess.opts.debugging_opts.emit_artifact_notifications {
if sess.opts.json_artifact_notifications {
sess.parse_sess.span_diagnostic
.emit_artifact_notification(&deps_filename, "dep-info");
}
@ -1059,7 +1059,7 @@ fn encode_and_write_metadata(
if let Err(e) = fs::rename(&metadata_filename, &out_filename) {
tcx.sess.fatal(&format!("failed to write {}: {}", out_filename.display(), e));
}
if tcx.sess.opts.debugging_opts.emit_artifact_notifications {
if tcx.sess.opts.json_artifact_notifications {
tcx.sess.parse_sess.span_diagnostic
.emit_artifact_notification(&out_filename, "metadata");
}

View File

@ -1071,7 +1071,7 @@ impl SaveHandler for DumpHandler<'_> {
error!("Can't serialize save-analysis: {:?}", e);
}
if sess.opts.debugging_opts.emit_artifact_notifications {
if sess.opts.json_artifact_notifications {
sess.parse_sess.span_diagnostic
.emit_artifact_notification(&file_name, "save-analysis");
}

View File

@ -3,10 +3,9 @@ use std::fmt;
use std::path::PathBuf;
use errors;
use errors::emitter::{ColorConfig, HumanReadableErrorType};
use getopts;
use rustc::lint::Level;
use rustc::session::early_error;
use rustc::session;
use rustc::session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs};
use rustc::session::config::{nightly_options, build_codegen_options, build_debugging_options,
get_cmd_lint_options, ExternEntry};
@ -243,36 +242,9 @@ impl Options {
return Err(0);
}
let color = match matches.opt_str("color").as_ref().map(|s| &s[..]) {
Some("auto") => ColorConfig::Auto,
Some("always") => ColorConfig::Always,
Some("never") => ColorConfig::Never,
None => ColorConfig::Auto,
Some(arg) => {
early_error(ErrorOutputType::default(),
&format!("argument for `--color` must be `auto`, `always` or `never` \
(instead was `{}`)", arg));
}
};
// FIXME: deduplicate this code from the identical code in librustc/session/config.rs
let error_format = match matches.opt_str("error-format").as_ref().map(|s| &s[..]) {
None |
Some("human") => ErrorOutputType::HumanReadable(HumanReadableErrorType::Default(color)),
Some("json") => ErrorOutputType::Json {
pretty: false,
json_rendered: HumanReadableErrorType::Default(ColorConfig::Never),
},
Some("pretty-json") => ErrorOutputType::Json {
pretty: true,
json_rendered: HumanReadableErrorType::Default(ColorConfig::Never),
},
Some("short") => ErrorOutputType::HumanReadable(HumanReadableErrorType::Short(color)),
Some(arg) => {
early_error(ErrorOutputType::default(),
&format!("argument for `--error-format` must be `human`, `json` or \
`short` (instead was `{}`)", arg));
}
};
let color = session::config::parse_color(&matches);
let (json_rendered, _artifacts) = session::config::parse_json(&matches);
let error_format = session::config::parse_error_format(&matches, color, json_rendered);
let codegen_options = build_codegen_options(matches, error_format);
let debugging_options = build_debugging_options(matches, error_format);

View File

@ -286,6 +286,12 @@ fn opts() -> Vec<RustcOptGroup> {
"How errors and other messages are produced",
"human|json|short")
}),
stable("json", |o| {
o.optopt("",
"json",
"Configure the structure of JSON diagnostics",
"CONFIG")
}),
unstable("disable-minification", |o| {
o.optflag("",
"disable-minification",

View File

@ -1,4 +1,4 @@
// compile-flags:--emit=metadata --error-format=json -Z emit-artifact-notifications
// compile-flags:--emit=metadata --error-format=json --json artifacts
// build-pass (FIXME(62277): could be check-pass?)
// ignore-pass
// ^-- needed because `--pass check` does not emit the output needed.

View File

@ -0,0 +1,4 @@
// build-fail
// compile-flags: --json=artifacts --error-format=json --color never
fn main() {}

View File

@ -0,0 +1,2 @@
error: cannot specify the `--color` option with `--json`

View File

@ -0,0 +1,4 @@
// build-fail
// compile-flags: --json=artifacts --error-format=short
fn main() {}

View File

@ -0,0 +1,2 @@
error: using `--json` requires also using `--error-format=json`

View File

@ -0,0 +1,4 @@
// build-fail
// compile-flags: --json=foo --error-format=json
fn main() {}

View File

@ -0,0 +1,2 @@
error: unknown `--json` option `foo`

View File

@ -0,0 +1 @@
{"artifact":"$TEST_BUILD_DIR/json-multiple.nll/libjson_multiple.rlib","emit":"link"}

View File

@ -0,0 +1,4 @@
// build-pass
// compile-flags: --json=diagnostic-short --json artifacts --error-format=json
#![crate_type = "lib"]

View File

@ -0,0 +1 @@
{"artifact":"$TEST_BUILD_DIR/json-multiple/libjson_multiple.rlib","emit":"link"}

View File

@ -0,0 +1 @@
{"artifact":"$TEST_BUILD_DIR/json-options.nll/libjson_options.rlib","emit":"link"}

View File

@ -0,0 +1,4 @@
// build-pass
// compile-flags: --json=diagnostic-short,artifacts --error-format=json
#![crate_type = "lib"]

View File

@ -0,0 +1 @@
{"artifact":"$TEST_BUILD_DIR/json-options/libjson_options.rlib","emit":"link"}

View File

@ -0,0 +1,2 @@
// build-fail
// compile-flags: --json=diagnostic-short --error-format=json

View File

@ -0,0 +1,19 @@
{"message":"`main` function not found in crate `json_short`","code":{"code":"E0601","explanation":"
No `main` function was found in a binary crate. To fix this error, add a
`main` function. For example:
```
fn main() {
// Your program will start here.
println!(\"Hello world!\");
}
```
If you don't know the basics of Rust, you can go look to the Rust Book to get
started: https://doc.rust-lang.org/book/
"},"level":"error","spans":[],"children":[{"message":"consider adding a `main` function to `$DIR/json-short.rs`","code":null,"level":"note","spans":[],"children":[],"rendered":null}],"rendered":"error[E0601]: `main` function not found in crate `json_short`
"}
{"message":"aborting due to previous error","code":null,"level":"error","spans":[],"children":[],"rendered":"error: aborting due to previous error
"}
{"message":"For more information about this error, try `rustc --explain E0601`.","code":null,"level":"","spans":[],"children":[],"rendered":"For more information about this error, try `rustc --explain E0601`.
"}

View File

@ -1,6 +1,6 @@
// ignore-cloudabi
// ignore-windows
// compile-flags: --error-format pretty-json -Zunstable-options --json-rendered=termcolor
// compile-flags: --error-format pretty-json --json=diagnostic-rendered-ansi
// The output for humans should just highlight the whole span without showing
// the suggested replacement, but we also want to test that suggested

View File

@ -73,8 +73,8 @@ mod foo {
"spans": [
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 484,
"byte_end": 488,
"byte_start": 471,
"byte_end": 475,
"line_start": 12,
"line_end": 12,
"column_start": 12,
@ -101,8 +101,8 @@ mod foo {
"spans": [
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -124,8 +124,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -147,8 +147,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -170,8 +170,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -193,8 +193,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -216,8 +216,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -239,8 +239,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -262,8 +262,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -285,8 +285,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -308,8 +308,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -331,8 +331,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,
@ -354,8 +354,8 @@ mod foo {
},
{
"file_name": "$DIR/use_suggestion_json.rs",
"byte_start": 461,
"byte_end": 461,
"byte_start": 448,
"byte_end": 448,
"line_start": 11,
"line_end": 11,
"column_start": 1,

View File

@ -1,5 +1,5 @@
// build-pass (FIXME(62277): could be check-pass?)
// compile-flags: -Zsave-analysis -Zemit-artifact-notifications
// compile-flags: -Zsave-analysis --json artifacts
// compile-flags: --crate-type rlib --error-format=json
// ignore-pass
// ^-- needed because otherwise, the .stderr file changes with --pass check