tvix_simstore/
lib.rs

1//! `tvix-simstore` implements a simulated Nix store implementation that can be “interacted” with
2//! from `tvix-eval`. This allows evaluating Nix expressions that use store dependent features
3//! like path interpolation, `builtins.derivation` etc. without even having the ability to write
4//! to a store let alone build a derivation. This is similar to the `dummy://` store implemented
5//! by C++ Nix (>= 2.4).
6//!
7//! Nix expressions that do need a functioning store, e.g. for import from derivation (IFD),
8//! will not work. To ensure purity, all reads from the store directory will result in
9//! [`SimulatedStoreError::StorePathRead`], i.e. `tvix-simstore` won't access store paths
10//! (i.e. paths below the configured `store_dir`) since they'd exist only by chance.
11//!
12//! Since no uniform store interface has been defined by `tvix-eval` yet, `tvix-simstore` consists
13//! of the following components:
14//!
15//! - [`SimulatedStoreIO`] implements the `EvalIO` trait and handles calculation of the store
16//!   paths for files that would need to be imported into the store.
17//! - The necessary additional builtins haven't been implemented yet.
18use std::borrow::Cow;
19use std::cell::RefCell;
20use std::collections::HashMap;
21use std::fmt;
22use std::fs;
23use std::io::{BufReader, Error, Read, Result};
24use std::iter::Peekable;
25use std::os::unix::ffi::OsStringExt;
26use std::os::unix::fs::MetadataExt;
27use std::path::{Path, PathBuf};
28
29use nix_compat::{
30    nar,
31    nixhash::{CAHash, NixHash},
32    store_path::{build_ca_path, StorePath},
33};
34use sha2::{Digest, Sha256};
35use tvix_eval::{builtin_macros::builtins, ErrorKind, EvalIO, FileType, StdIO, Value};
36
37pub struct SimulatedStoreIO {
38    store_dir: String,
39    passthru_paths: RefCell<HashMap<[u8; 20], PathBuf>>,
40}
41
42// TODO: copied from glue/import.rs; where should this live?
43fn path_to_name(path: &Path) -> std::io::Result<&str> {
44    path.file_name()
45        .and_then(|file_name| file_name.to_str())
46        .ok_or_else(|| {
47            std::io::Error::new(
48                std::io::ErrorKind::InvalidInput,
49                "path must not be .. and the basename valid unicode",
50            )
51        })
52}
53
54impl SimulatedStoreIO {
55    /// Adds a passthru path, mapping the given path to the given location on the
56    /// filesystem.
57    ///
58    /// Using this incorrectly can lead to incomprehensible breakage.
59    pub fn add_passthru(&mut self, path: &str, loc: PathBuf) -> Result<()> {
60        let (store_path, _) =
61            StorePath::<&str>::from_absolute_path_full(path).map_err(Error::other)?;
62
63        self.passthru_paths
64            .borrow_mut()
65            .insert(*store_path.digest(), loc);
66        Ok(())
67    }
68}
69
70#[derive(Debug, PartialEq, Eq)]
71pub enum SimulatedStoreError {
72    StorePathRead,
73    NixCompatError(nix_compat::store_path::Error),
74}
75
76impl fmt::Display for SimulatedStoreError {
77    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
78        match self {
79            SimulatedStoreError::StorePathRead => {
80                write!(f, "simstore would need to read from a realised store path")
81            }
82
83            SimulatedStoreError::NixCompatError(cause) => {
84                write!(f, "invalid Nix store path: ")?;
85                cause.fmt(f)
86            }
87        }
88    }
89}
90
91impl std::error::Error for SimulatedStoreError {
92    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
93        None
94    }
95}
96
97impl From<nix_compat::store_path::Error> for SimulatedStoreError {
98    fn from(cause: nix_compat::store_path::Error) -> Self {
99        Self::NixCompatError(cause)
100    }
101}
102
103impl Default for SimulatedStoreIO {
104    fn default() -> Self {
105        Self {
106            store_dir: "/nix/store".to_owned(),
107            passthru_paths: Default::default(),
108        }
109    }
110}
111
112// TODO(sterni): creation with configurable store dir
113impl SimulatedStoreIO {
114    /// Returns a path from which StdIO can read, unless realisation is required
115    /// (which the simulated store does not support).
116    fn to_readable_path<'a>(&self, path: &'a Path) -> Result<Cow<'a, Path>> {
117        if !path.starts_with(Path::new(&self.store_dir)) {
118            return Ok(Cow::Borrowed(path));
119        }
120
121        let (store_path, relative) =
122            StorePath::<&str>::from_absolute_path_full(path).map_err(Error::other)?;
123
124        // Pass known imported paths through to their original locations.
125        if let Some(base) = self.passthru_paths.borrow().get(store_path.digest()) {
126            return Ok(Cow::Owned(if relative.as_os_str().is_empty() {
127                base.into()
128            } else {
129                base.join(relative)
130            }));
131        }
132
133        // Allow reads from the "real" Nix store locally.
134        if StdIO.path_exists(path)? {
135            return Ok(Cow::Borrowed(path));
136        }
137
138        Err(Error::other(SimulatedStoreError::StorePathRead))
139    }
140
141    pub fn import_path_by_entries<I, E>(
142        &self,
143        name: &str,
144        entries: I,
145        expected_sha256: Option<[u8; 32]>,
146    ) -> Result<StorePath<String>>
147    where
148        Error: From<E>,
149        I: Iterator<Item = std::result::Result<walkdir::DirEntry, E>>,
150    {
151        let mut hash = Sha256::new();
152        let nar = nar::writer::open(&mut hash)?;
153
154        pack_entries(nar, &mut entries.peekable())?;
155
156        let nar_hash = NixHash::Sha256(hash.finalize().into());
157
158        if let Some(expected) = expected_sha256 {
159            if nar_hash != NixHash::Sha256(expected) {
160                // TODO: this error is really bad; needs both hashes etc.
161                // It doesn't feel like this is the right place.
162                return Err(Error::other("expected hash does not match"));
163            }
164        }
165
166        let hash = CAHash::Nar(nar_hash);
167        build_ca_path(name, &hash, Option::<String>::default(), false).map_err(Error::other)
168    }
169}
170
171fn pack_entries_dir<W, E, I>(
172    mut dir: nar::writer::Directory<'_, W>,
173    depth: usize,
174    walker: &mut Peekable<I>,
175) -> Result<()>
176where
177    W: std::io::Write,
178    Error: From<E>,
179    I: Iterator<Item = std::result::Result<walkdir::DirEntry, E>>,
180{
181    loop {
182        let peeked = match walker.peek() {
183            None => break,
184            Some(e) => e,
185        };
186
187        // `peeked` borrows the next result, if it is an error we need to
188        // "actually" take it to be able to propagate the error.
189        let entry = match peeked {
190            Ok(e) => e,
191            Err(_) => {
192                walker.next().expect("is present")?;
193                unreachable!("above `?` always exits");
194            }
195        };
196
197        if entry.depth() < depth {
198            break;
199        }
200
201        let nar = dir.entry(entry.file_name().to_owned().into_vec().as_slice())?;
202        pack_entries(nar, walker)?;
203    }
204
205    dir.close()?;
206
207    Ok(())
208}
209
210fn pack_entries<W, E, I>(nar: nar::writer::Node<'_, W>, walker: &mut Peekable<I>) -> Result<()>
211where
212    W: std::io::Write,
213    Error: From<E>,
214    I: Iterator<Item = std::result::Result<walkdir::DirEntry, E>>,
215{
216    let entry = if let Some(entry) = walker.next() {
217        entry?
218    } else {
219        return Ok(());
220    };
221
222    let ft = entry.file_type();
223    if ft.is_symlink() {
224        let target = fs::read_link(entry.path())?.into_os_string();
225        nar.symlink(target.into_vec().as_slice())?;
226    } else if ft.is_file() {
227        let meta = entry.metadata()?;
228        let executable = (meta.mode() & 0o100) != 0;
229        let file = fs::File::open(entry.path())?;
230        let mut reader = BufReader::new(file);
231        nar.file(executable, meta.size(), &mut reader)?;
232    } else if ft.is_dir() {
233        let inner_depth = entry.depth() + 1;
234        let dir = nar.directory()?;
235        pack_entries_dir(dir, inner_depth, walker)?;
236    } else {
237        return Err(Error::new(
238            std::io::ErrorKind::InvalidData,
239            "invalid file type for store ingestion",
240        ));
241    }
242
243    Ok(())
244}
245
246impl EvalIO for SimulatedStoreIO {
247    fn store_dir(&self) -> Option<String> {
248        Some(self.store_dir.clone())
249    }
250
251    fn import_path(&self, path: &Path) -> Result<PathBuf> {
252        let path = path.canonicalize()?;
253        let mut hash = Sha256::new();
254        let nar = nar::writer::open(&mut hash)?;
255
256        let walker = walkdir::WalkDir::new(path.clone())
257            .follow_links(false)
258            .follow_root_links(false)
259            .contents_first(false)
260            .sort_by(|a, b| a.file_name().cmp(b.file_name()))
261            .into_iter();
262
263        pack_entries(nar, &mut walker.peekable())?;
264
265        let name = path_to_name(&path)?;
266        let hash = CAHash::Nar(NixHash::Sha256(hash.finalize().into()));
267        let store_path: StorePath<&str> =
268            build_ca_path(name, &hash, Option::<&str>::default(), false).map_err(Error::other)?;
269
270        self.passthru_paths
271            .borrow_mut()
272            .insert(*store_path.digest(), path.to_owned());
273
274        Ok(PathBuf::from(store_path.to_absolute_path()))
275    }
276
277    // TODO(sterni): proc macro for dispatching methods
278    fn path_exists(&self, path: &Path) -> Result<bool> {
279        StdIO.path_exists(self.to_readable_path(path)?.as_ref())
280    }
281
282    fn open(&self, path: &Path) -> Result<Box<dyn Read>> {
283        StdIO.open(self.to_readable_path(path)?.as_ref())
284    }
285
286    fn file_type(&self, path: &Path) -> Result<FileType> {
287        StdIO.file_type(self.to_readable_path(path)?.as_ref())
288    }
289
290    fn read_dir(&self, path: &Path) -> Result<Vec<(bytes::Bytes, FileType)>> {
291        StdIO.read_dir(self.to_readable_path(path)?.as_ref())
292    }
293}
294
295// TODO(sterni): implement simulation, parse args
296// TODO(sterni): move derivationStrict simulation here
297#[builtins]
298mod builtins {
299    use super::*;
300    use tvix_eval::generators::{Gen, GenCo};
301
302    #[builtin("fetchGit")]
303    async fn builtin_fetch_git(co: GenCo, args: Value) -> std::result::Result<Value, ErrorKind> {
304        Err(ErrorKind::NotImplemented("fetchGit"))
305    }
306
307    #[builtin("fetchMercurial")]
308    async fn builtin_fetch_mercurial(
309        co: GenCo,
310        args: Value,
311    ) -> std::result::Result<Value, ErrorKind> {
312        Err(ErrorKind::NotImplemented("fetchMercurial"))
313    }
314
315    #[builtin("fetchTarball")]
316    async fn builtin_fetch_tarball(
317        co: GenCo,
318        args: Value,
319    ) -> std::result::Result<Value, ErrorKind> {
320        Err(ErrorKind::NotImplemented("fetchTarball"))
321    }
322}
323
324pub fn simulated_store_builtins() -> Vec<(&'static str, Value)> {
325    builtins::builtins()
326}
327
328#[cfg(test)]
329mod tests {
330    use super::*;
331
332    fn io_err_to_simstore_err<T>(res: Result<T>) -> SimulatedStoreError {
333        res.err()
334            .expect("Store Path Read should fail")
335            .downcast::<SimulatedStoreError>()
336            .expect("Should be SimulatedStoreError")
337    }
338
339    #[test]
340    fn store_read_disallowed() {
341        let paths = vec![
342            "0a00kbgj7n5s2ds6r2ffsmbz8rkg3hdh-charset-0.3.10-r1.cabal.drv",
343            "bz02y8zr6wp4yli9bqadjmf0biqinw6i-website/index.html",
344            "n2v8qwc85kl4nk6ipfpaxs2pkjzka4v8-serve-examples",
345        ];
346        let store_io = SimulatedStoreIO::default();
347
348        for path in paths {
349            let mut abs = PathBuf::from(
350                store_io
351                    .store_dir()
352                    .expect("SimulatedStoreIO should have a store_dir"),
353            );
354            abs.push(path);
355
356            assert!(store_io.to_readable_path(&abs).is_err());
357
358            assert_eq!(
359                io_err_to_simstore_err(store_io.path_exists(&abs)),
360                SimulatedStoreError::StorePathRead
361            );
362            assert_eq!(
363                io_err_to_simstore_err(store_io.file_type(&abs)),
364                SimulatedStoreError::StorePathRead
365            );
366            assert_eq!(
367                io_err_to_simstore_err(store_io.open(&abs)),
368                SimulatedStoreError::StorePathRead
369            );
370            assert_eq!(
371                io_err_to_simstore_err(store_io.read_dir(&abs)),
372                SimulatedStoreError::StorePathRead
373            );
374        }
375    }
376
377    #[test]
378    fn imported_paths() {
379        let store_io = SimulatedStoreIO::default();
380        assert_eq!(
381            store_io
382                .import_path(Path::new("./test-data/q.txt"))
383                .expect("importing test data should succeed"),
384            Path::new("/nix/store/6w97x3p5yw17nwvqn3s6mrhdlznmzmiv-q.txt")
385        );
386        assert_eq!(
387            store_io
388                .import_path(Path::new("./test-data"))
389                .expect("importing test data should succeed"),
390            Path::new("/nix/store/ljqm0pf4b43bk53lymzrbljvdxi5vkcm-test-data")
391        );
392    }
393
394    #[test]
395    fn passthru_paths_file() {
396        let store_io = SimulatedStoreIO::default();
397        let imported = store_io
398            .import_path(Path::new("./test-data/q.txt"))
399            .expect("importing test data should succeed");
400        assert!(store_io
401            .path_exists(&imported)
402            .expect("imported path should be forwarded"));
403    }
404
405    #[test]
406    fn passthru_paths_folder() {
407        let store_io = SimulatedStoreIO::default();
408        let imported = store_io
409            .import_path(Path::new("./test-data"))
410            .expect("importing test data should succeed");
411        assert!(store_io
412            .path_exists(&imported.join("q.txt"))
413            .expect("imported path should be forwarded"));
414    }
415
416    #[test]
417    fn added_passthru_path() {
418        let mut store_io = SimulatedStoreIO::default();
419        let example = "/nix/store/a396z42saqql55cp5n1vrb2j0siq86k1-nixpkgs-src";
420        let example_path = PathBuf::from(example);
421
422        store_io
423            .add_passthru(example, example_path.clone())
424            .expect("adding passthru should work");
425
426        store_io
427            .path_exists(&example_path)
428            .expect("path access should not fail");
429    }
430}
tvix_simstore/lib.rs

tvix_simstore/
lib.rs
