dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #122070 - Zoxc:dep-edges-from-previous, r=cjgillot 

Encode dep graph edges directly from the previous graph when promoting

This encodes dep graph edges directly from the previous graph when promoting nodes from a previous session, avoiding allocations / copies.

~~Based on https://github.com/rust-lang/rust/pull/122064 and https://github.com/rust-lang/rust/pull/116375.~~

<table><tr><td rowspan="2">Benchmark</td><td colspan="1"><b>Before</b></th><td colspan="2"><b>After</b></th></tr><tr><td align="right">Time</td><td align="right">Time</td><td align="right">%</th></tr><tr><td>🟣 <b>clap</b>:check:unchanged</td><td align="right">0.4177s</td><td align="right">0.4072s</td><td align="right">💚  -2.52%</td></tr><tr><td>🟣 <b>hyper</b>:check:unchanged</td><td align="right">0.1430s</td><td align="right">0.1420s</td><td align="right"> -0.69%</td></tr><tr><td>🟣 <b>regex</b>:check:unchanged</td><td align="right">0.3106s</td><td align="right">0.3038s</td><td align="right">💚  -2.19%</td></tr><tr><td>🟣 <b>syn</b>:check:unchanged</td><td align="right">0.5823s</td><td align="right">0.5688s</td><td align="right">💚  -2.33%</td></tr><tr><td>🟣 <b>syntex_syntax</b>:check:unchanged</td><td align="right">1.3992s</td><td align="right">1.3692s</td><td align="right">💚  -2.14%</td></tr><tr><td>Total</td><td align="right">2.8528s</td><td align="right">2.7910s</td><td align="right">💚  -2.17%</td></tr><tr><td>Summary</td><td align="right">1.0000s</td><td align="right">0.9803s</td><td align="right">💚  -1.97%</td></tr></table>
This commit is contained in:
bors 2024-04-05 11:11:17 +00:00
parent d009f60b55 aa9c9a36c0
commit 4563f70c3b
4 changed files with 189 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
compiler/rustc_incremental/src/persist/load.rs
View File

@ -11,6 +11,7 @@ use rustc_session::config::IncrementalStateAssertion;
use rustc_session::Session;
use rustc_span::ErrorGuaranteed;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use super::data::*;
use super::file_format;
@ -88,7 +89,7 @@ fn delete_dirty_work_product(sess: &Session, swp: SerializedWorkProduct) {
work_product::delete_workproduct_files(sess, &swp.work_product);
}
fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProductMap)> {
fn load_dep_graph(sess: &Session) -> LoadResult<(Arc<SerializedDepGraph>, WorkProductMap)> {
let prof = sess.prof.clone();
if sess.opts.incremental.is_none() {

3
compiler/rustc_incremental/src/persist/save.rs
View File

@ -10,6 +10,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
use rustc_serialize::Encodable as RustcEncodable;
use rustc_session::Session;
use std::fs;
use std::sync::Arc;
use super::data::*;
use super::dirty_clean;
@ -147,7 +148,7 @@ fn encode_query_cache(tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult
/// and moves it to the permanent dep-graph path
pub(crate) fn build_dep_graph(
sess: &Session,
prev_graph: SerializedDepGraph,
prev_graph: Arc<SerializedDepGraph>,
prev_work_products: WorkProductMap,
) -> Option<DepGraph> {
if sess.opts.incremental.is_none() {

22
compiler/rustc_query_system/src/dep_graph/graph.rs
View File

@ -13,6 +13,7 @@ use std::fmt::Debug;
use std::hash::Hash;
use std::marker::PhantomData;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use super::query::DepGraphQuery;
use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
@ -81,7 +82,7 @@ pub(crate) struct DepGraphData<D: Deps> {
/// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
previous: SerializedDepGraph,
previous: Arc<SerializedDepGraph>,
colors: DepNodeColorMap,
@ -113,7 +114,7 @@ where
impl<D: Deps> DepGraph<D> {
pub fn new(
profiler: &SelfProfilerRef,
prev_graph: SerializedDepGraph,
prev_graph: Arc<SerializedDepGraph>,
prev_work_products: WorkProductMap,
encoder: FileEncoder,
record_graph: bool,
@ -127,6 +128,7 @@ impl<D: Deps> DepGraph<D> {
encoder,
record_graph,
record_stats,
prev_graph.clone(),
);
let colors = DepNodeColorMap::new(prev_graph_node_count);
@ -1084,6 +1086,7 @@ impl<D: Deps> CurrentDepGraph<D> {
encoder: FileEncoder,
record_graph: bool,
record_stats: bool,
previous: Arc<SerializedDepGraph>,
) -> Self {
use std::time::{SystemTime, UNIX_EPOCH};
@ -1116,6 +1119,7 @@ impl<D: Deps> CurrentDepGraph<D> {
record_graph,
record_stats,
profiler,
previous,
),
new_node_to_index: Sharded::new(|| {
FxHashMap::with_capacity_and_hasher(
@ -1236,16 +1240,14 @@ impl<D: Deps> CurrentDepGraph<D> {
match prev_index_to_index[prev_index] {
Some(dep_node_index) => dep_node_index,
None => {
let key = prev_graph.index_to_node(prev_index);
let edges = prev_graph
.edge_targets_from(prev_index)
.map(|i| prev_index_to_index[i].unwrap())
.collect();
let fingerprint = prev_graph.fingerprint_by_index(prev_index);
let dep_node_index = self.encoder.send(key, fingerprint, edges);
let dep_node_index = self.encoder.send_promoted(prev_index, &*prev_index_to_index);
prev_index_to_index[prev_index] = Some(dep_node_index);
#[cfg(debug_assertions)]
self.record_edge(dep_node_index, key, fingerprint);
self.record_edge(
dep_node_index,
prev_graph.index_to_node(prev_index),
prev_graph.fingerprint_by_index(prev_index),
);
dep_node_index
}
}

212
compiler/rustc_query_system/src/dep_graph/serialized.rs
View File

@ -41,6 +41,7 @@ use crate::dep_graph::edges::EdgesVec;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::fingerprint::PackedFingerprint;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::outline;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::sync::Lock;
use rustc_data_structures::unhash::UnhashMap;
@ -49,6 +50,7 @@ use rustc_serialize::opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixed
use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
use std::iter;
use std::marker::PhantomData;
use std::sync::Arc;
// The maximum value of `SerializedDepNodeIndex` leaves the upper two bits
// unused so that we can store multiple index types in `CompressedHybridIndex`,
@ -94,7 +96,7 @@ impl SerializedDepGraph {
pub fn edge_targets_from(
&self,
source: SerializedDepNodeIndex,
) -> impl Iterator<Item = SerializedDepNodeIndex> + '_ {
) -> impl Iterator<Item = SerializedDepNodeIndex> + Clone + '_ {
let header = self.edge_list_indices[source];
let mut raw = &self.edge_list_data[header.start()..];
// Figure out where the edge list for `source` ends by getting the start index of the next
@ -176,7 +178,7 @@ fn mask(bits: usize) -> usize {
impl SerializedDepGraph {
#[instrument(level = "debug", skip(d))]
pub fn decode<D: Deps>(d: &mut MemDecoder<'_>) -> SerializedDepGraph {
pub fn decode<D: Deps>(d: &mut MemDecoder<'_>) -> Arc<SerializedDepGraph> {
// The last 16 bytes are the node count and edge count.
debug!("position: {:?}", d.position());
let (node_count, edge_count, graph_size) =
@ -254,7 +256,13 @@ impl SerializedDepGraph {
index[node.kind.as_usize()].insert(node.hash, idx);
}
SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index }
Arc::new(SerializedDepGraph {
nodes,
fingerprints,
edge_list_indices,
edge_list_data,
index,
})
}
}
@ -299,21 +307,24 @@ impl<D: Deps> SerializedNodeHeader<D> {
const MAX_INLINE_LEN: usize = (u16::MAX as usize >> (Self::TOTAL_BITS - Self::LEN_BITS)) - 1;
#[inline]
fn new(node_info: &NodeInfo) -> Self {
fn new(
node: DepNode,
fingerprint: Fingerprint,
edge_max_index: u32,
edge_count: usize,
) -> Self {
debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS);
let NodeInfo { node, fingerprint, edges } = node_info;
let mut head = node.kind.as_inner();
let free_bytes = edges.max_index().leading_zeros() as usize / 8;
let free_bytes = edge_max_index.leading_zeros() as usize / 8;
let bytes_per_index = (DEP_NODE_SIZE - free_bytes).saturating_sub(1);
head |= (bytes_per_index as u16) << Self::KIND_BITS;
// Encode number of edges + 1 so that we can reserve 0 to indicate that the len doesn't fit
// in this bitfield.
if edges.len() <= Self::MAX_INLINE_LEN {
head |= (edges.len() as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
if edge_count <= Self::MAX_INLINE_LEN {
head |= (edge_count as u16 + 1) << (Self::KIND_BITS + Self::WIDTH_BITS);
}
let hash: Fingerprint = node.hash.into();
@ -327,10 +338,10 @@ impl<D: Deps> SerializedNodeHeader<D> {
#[cfg(debug_assertions)]
{
let res = Self { bytes, _marker: PhantomData };
assert_eq!(node_info.fingerprint, res.fingerprint());
assert_eq!(node_info.node, res.node());
assert_eq!(fingerprint, res.fingerprint());
assert_eq!(node, res.node());
if let Some(len) = res.len() {
assert_eq!(node_info.edges.len(), len);
assert_eq!(edge_count, len);
}
}
Self { bytes, _marker: PhantomData }
@ -393,21 +404,61 @@ struct NodeInfo {
impl NodeInfo {
fn encode<D: Deps>(&self, e: &mut FileEncoder) {
let header = SerializedNodeHeader::<D>::new(self);
let NodeInfo { node, fingerprint, ref edges } = *self;
let header =
SerializedNodeHeader::<D>::new(node, fingerprint, edges.max_index(), edges.len());
e.write_array(header.bytes);
if header.len().is_none() {
e.emit_usize(self.edges.len());
e.emit_usize(edges.len());
}
let bytes_per_index = header.bytes_per_index();
for node_index in self.edges.iter() {
for node_index in edges.iter() {
e.write_with(|dest| {
*dest = node_index.as_u32().to_le_bytes();
bytes_per_index
});
}
}
/// Encode a node that was promoted from the previous graph. It reads the edges directly from
/// the previous dep graph and expects all edges to already have a new dep node index assigned.
/// This avoids the overhead of constructing `EdgesVec`, which would be needed to call `encode`.
#[inline]
fn encode_promoted<D: Deps>(
e: &mut FileEncoder,
node: DepNode,
fingerprint: Fingerprint,
prev_index: SerializedDepNodeIndex,
prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
previous: &SerializedDepGraph,
) -> usize {
let edges = previous.edge_targets_from(prev_index);
let edge_count = edges.size_hint().0;
// Find the highest edge in the new dep node indices
let edge_max =
edges.clone().map(|i| prev_index_to_index[i].unwrap().as_u32()).max().unwrap_or(0);
let header = SerializedNodeHeader::<D>::new(node, fingerprint, edge_max, edge_count);
e.write_array(header.bytes);
if header.len().is_none() {
e.emit_usize(edge_count);
}
let bytes_per_index = header.bytes_per_index();
for node_index in edges {
let node_index = prev_index_to_index[node_index].unwrap();
e.write_with(|dest| {
*dest = node_index.as_u32().to_le_bytes();
bytes_per_index
});
}
edge_count
}
}
struct Stat {
@ -417,6 +468,7 @@ struct Stat {
}
struct EncoderState<D: Deps> {
previous: Arc<SerializedDepGraph>,
encoder: FileEncoder,
total_node_count: usize,
total_edge_count: usize,
@ -428,8 +480,9 @@ struct EncoderState<D: Deps> {
}
impl<D: Deps> EncoderState<D> {
fn new(encoder: FileEncoder, record_stats: bool) -> Self {
fn new(encoder: FileEncoder, record_stats: bool, previous: Arc<SerializedDepGraph>) -> Self {
Self {
previous,
encoder,
total_edge_count: 0,
total_node_count: 0,
@ -439,36 +492,99 @@ impl<D: Deps> EncoderState<D> {
}
}
#[inline]
fn record(
&mut self,
node: DepNode,
edge_count: usize,
edges: impl FnOnce(&mut Self) -> Vec<DepNodeIndex>,
record_graph: &Option<Lock<DepGraphQuery>>,
) -> DepNodeIndex {
let index = DepNodeIndex::new(self.total_node_count);
self.total_node_count += 1;
self.kind_stats[node.kind.as_usize()] += 1;
self.total_edge_count += edge_count;
if let Some(record_graph) = &record_graph {
// Call `edges` before the outlined code to allow the closure to be optimized out.
let edges = edges(self);
// Outline the build of the full dep graph as it's typically disabled and cold.
outline(move || {
// Do not ICE when a query is called from within `with_query`.
if let Some(record_graph) = &mut record_graph.try_lock() {
record_graph.push(index, node, &edges);
}
});
}
if let Some(stats) = &mut self.stats {
let kind = node.kind;
// Outline the stats code as it's typically disabled and cold.
outline(move || {
let stat =
stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
});
}
index
}
/// Encodes a node to the current graph.
fn encode_node(
&mut self,
node: &NodeInfo,
record_graph: &Option<Lock<DepGraphQuery>>,
) -> DepNodeIndex {
let index = DepNodeIndex::new(self.total_node_count);
self.total_node_count += 1;
self.kind_stats[node.node.kind.as_usize()] += 1;
node.encode::<D>(&mut self.encoder);
self.record(
node.node,
node.edges.len(),
|_| node.edges[..].iter().copied().collect(),
record_graph,
)
}
let edge_count = node.edges.len();
self.total_edge_count += edge_count;
/// Encodes a node that was promoted from the previous graph. It reads the information directly from
/// the previous dep graph for performance reasons.
///
/// This differs from `encode_node` where you have to explictly provide the relevant `NodeInfo`.
///
/// It expects all edges to already have a new dep node index assigned.
#[inline]
fn encode_promoted_node(
&mut self,
prev_index: SerializedDepNodeIndex,
record_graph: &Option<Lock<DepGraphQuery>>,
prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
) -> DepNodeIndex {
let node = self.previous.index_to_node(prev_index);
if let Some(record_graph) = &record_graph {
// Do not ICE when a query is called from within `with_query`.
if let Some(record_graph) = &mut record_graph.try_lock() {
record_graph.push(index, node.node, &node.edges);
}
}
let fingerprint = self.previous.fingerprint_by_index(prev_index);
let edge_count = NodeInfo::encode_promoted::<D>(
&mut self.encoder,
node,
fingerprint,
prev_index,
prev_index_to_index,
&self.previous,
);
if let Some(stats) = &mut self.stats {
let kind = node.node.kind;
let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 });
stat.node_counter += 1;
stat.edge_counter += edge_count as u64;
}
let encoder = &mut self.encoder;
node.encode::<D>(encoder);
index
self.record(
node,
edge_count,
|this| {
this.previous
.edge_targets_from(prev_index)
.map(|i| prev_index_to_index[i].unwrap())
.collect()
},
record_graph,
)
}
fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult {
@ -479,6 +595,7 @@ impl<D: Deps> EncoderState<D> {
stats: _,
kind_stats,
marker: _,
previous: _,
} = self;
let node_count = total_node_count.try_into().unwrap();
@ -520,9 +637,10 @@ impl<D: Deps> GraphEncoder<D> {
record_graph: bool,
record_stats: bool,
profiler: &SelfProfilerRef,
previous: Arc<SerializedDepGraph>,
) -> Self {
let record_graph = record_graph.then(|| Lock::new(DepGraphQuery::new(prev_node_count)));
let status = Lock::new(Some(EncoderState::new(encoder, record_stats)));
let status = Lock::new(Some(EncoderState::new(encoder, record_stats, previous)));
GraphEncoder { status, record_graph, profiler: profiler.clone() }
}
@ -596,6 +714,22 @@ impl<D: Deps> GraphEncoder<D> {
self.status.lock().as_mut().unwrap().encode_node(&node, &self.record_graph)
}
/// Encodes a node that was promoted from the previous graph. It reads the information directly from
/// the previous dep graph and expects all edges to already have a new dep node index assigned.
#[inline]
pub(crate) fn send_promoted(
&self,
prev_index: SerializedDepNodeIndex,
prev_index_to_index: &IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>,
) -> DepNodeIndex {
let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph");
self.status.lock().as_mut().unwrap().encode_promoted_node(
prev_index,
&self.record_graph,
prev_index_to_index,
)
}
pub fn finish(&self) -> FileEncodeResult {
let _prof_timer = self.profiler.generic_activity("incr_comp_encode_dep_graph_finish");