tvix_glue/builtins/
import.rs

1//! Implements builtins used to import paths in the store.
2
3use std::path::Path;
4use std::rc::Rc;
5
6use crate::tvix_store_io::TvixStoreIO;
7use nix_compat::store_path::{build_ca_path, StorePath, StorePathRef};
8use tvix_eval::{
9    builtin_macros::builtins,
10    generators::{self, GenCo},
11    ErrorKind, EvalIO, Value,
12};
13
14/// Transform a path into its base name and returns an [`std::io::Error`] if it is `..` or if the
15/// basename is not valid unicode.
16fn path_to_name(path: &Path) -> std::io::Result<&str> {
17    path.file_name()
18        .and_then(|file_name| file_name.to_str())
19        .ok_or_else(|| {
20            std::io::Error::new(
21                std::io::ErrorKind::InvalidInput,
22                "path must not be .. and the basename valid unicode",
23            )
24        })
25}
26
27async fn filtered_ingest(
28    state: Rc<TvixStoreIO>,
29    co: GenCo,
30    path: &Path,
31    name: Option<String>,
32    filter: Option<&Value>,
33) -> Result<StorePath<String>, ErrorKind> {
34    let mut entries: Vec<walkdir::DirEntry> = vec![];
35    let mut it = walkdir::WalkDir::new(path)
36        .follow_links(false)
37        .follow_root_links(false)
38        .contents_first(false)
39        .sort_by(|a, b| a.file_name().cmp(b.file_name()))
40        .into_iter();
41
42    // Always add root node.
43    entries.push(
44        it.next()
45            .ok_or_else(|| ErrorKind::IO {
46                path: Some(path.to_path_buf()),
47                error: std::io::Error::new(std::io::ErrorKind::NotFound, "No root node emitted")
48                    .into(),
49            })?
50            .map_err(|err| ErrorKind::IO {
51                path: Some(path.to_path_buf()),
52                error: std::io::Error::from(err).into(),
53            })?,
54    );
55
56    while let Some(entry) = it.next() {
57        // Entry could be a NotFound, if the root path specified does not exist.
58        let entry = entry.map_err(|err| ErrorKind::IO {
59            path: err.path().map(|p| p.to_path_buf()),
60            error: std::io::Error::from(err).into(),
61        })?;
62
63        // As per Nix documentation `:doc builtins.filterSource`.
64        let file_type = if entry.file_type().is_dir() {
65            "directory"
66        } else if entry.file_type().is_file() {
67            "regular"
68        } else if entry.file_type().is_symlink() {
69            "symlink"
70        } else {
71            "unknown"
72        };
73
74        let should_keep: bool = if let Some(filter) = filter {
75            generators::request_force(
76                &co,
77                generators::request_call_with(
78                    &co,
79                    filter.clone(),
80                    [
81                        Value::String(entry.path().as_os_str().as_encoded_bytes().into()),
82                        Value::String(file_type.into()),
83                    ],
84                )
85                .await,
86            )
87            .await
88            .as_bool()?
89        } else {
90            true
91        };
92
93        if !should_keep {
94            if file_type == "directory" {
95                it.skip_current_dir();
96            }
97            continue;
98        }
99
100        entries.push(entry);
101    }
102
103    let dir_entries = entries
104        .into_iter()
105        .map(Ok::<walkdir::DirEntry, std::io::Error>);
106
107    let name = match name {
108        Some(name) => name,
109        None => path_to_name(path)
110            .expect("failed to derive the default name out of the path")
111            .to_string(),
112    };
113
114    Ok(state
115        .simulated_store
116        .import_path_by_entries(&name, dir_entries, None)?)
117}
118
119#[builtins(state = "Rc<TvixStoreIO>")]
120mod import_builtins {
121    use super::*;
122
123    use crate::builtins::ImportError;
124    use crate::tvix_store_io::TvixStoreIO;
125    use bstr::ByteSlice;
126    use nix_compat::nixhash::{CAHash, NixHash};
127    use sha2::Digest;
128    use std::rc::Rc;
129    use tvix_eval::builtins::coerce_value_to_path;
130    use tvix_eval::generators::Gen;
131    use tvix_eval::{generators::GenCo, ErrorKind, Value};
132    use tvix_eval::{AddContext, FileType, NixContext, NixContextElement, NixString};
133
134    // This is a helper used by both builtins.path and builtins.filterSource.
135    async fn import_helper(
136        state: Rc<TvixStoreIO>,
137        co: GenCo,
138        path: std::path::PathBuf,
139        name: Option<&Value>,
140        filter: Option<&Value>,
141        recursive_ingestion: bool,
142        expected_sha256: Option<[u8; 32]>,
143    ) -> Result<Value, ErrorKind> {
144        let name: String = match name {
145            Some(name) => generators::request_force(&co, name.clone())
146                .await
147                .to_str()?
148                .as_bstr()
149                .to_string(),
150
151            None => path_to_name(&path)
152                .expect("Failed to derive the default name out of the path")
153                .to_string(),
154        };
155
156        let store_path = match std::fs::metadata(&path)?.file_type().into() {
157            // Regular file, non-recursive -> ingest with plain SHA256 content hash
158            FileType::Regular if !recursive_ingestion => {
159                let mut file = state.open(&path)?;
160                let mut hasher = sha2::Sha256::new();
161                let mut buffer = [0; 8192]; // 8KB buffer is a reasonable size \/(O.o)\/
162
163                loop {
164                    let bytes_read = file.read(&mut buffer)?;
165                    if bytes_read == 0 {
166                        break;
167                    }
168                    hasher.update(&buffer[..bytes_read]);
169                }
170
171                let actual_sha256 = hasher.finalize().into();
172
173                // If an expected hash was provided upfront, compare and bail out.
174                if let Some(expected_sha256) = expected_sha256 {
175                    if actual_sha256 != expected_sha256 {
176                        return Err(ImportError::HashMismatch(
177                            path.clone(),
178                            NixHash::Sha256(expected_sha256),
179                            NixHash::Sha256(actual_sha256),
180                        )
181                        .into());
182                    }
183                }
184
185                let ca = CAHash::Flat(NixHash::Sha256(actual_sha256));
186                build_ca_path(&name, &ca, Vec::<&str>::new(), false)
187                    .map_err(|e| tvix_eval::ErrorKind::TvixError(Rc::new(e)))?
188            }
189
190            FileType::Regular => {
191                let dir_entry = walkdir::WalkDir::new(path)
192                    .follow_root_links(false)
193                    .into_iter();
194
195                state
196                    .simulated_store
197                    .import_path_by_entries(&name, dir_entry, expected_sha256)?
198            }
199
200            FileType::Directory if !recursive_ingestion => {
201                return Err(ImportError::FlatImportOfNonFile(path))?
202            }
203
204            // do the filtered ingest
205            FileType::Directory => {
206                filtered_ingest(state.clone(), co, path.as_ref(), Some(name), filter).await?
207            }
208
209            FileType::Symlink => {
210                // FUTUREWORK: Nix follows a symlink if it's at the root,
211                // except if it's not resolve-able (NixOS/nix#7761).i
212                return Err(tvix_eval::ErrorKind::IO {
213                    path: Some(path),
214                    error: Rc::new(std::io::Error::new(
215                        std::io::ErrorKind::Unsupported,
216                        "builtins.path pointing to a symlink is ill-defined.",
217                    )),
218                });
219            }
220            FileType::Unknown => {
221                return Err(tvix_eval::ErrorKind::IO {
222                    path: Some(path),
223                    error: Rc::new(std::io::Error::new(
224                        std::io::ErrorKind::Unsupported,
225                        "unsupported file type",
226                    )),
227                })
228            }
229        };
230
231        let outpath = store_path.to_absolute_path();
232        let ctx: NixContext = NixContextElement::Plain(outpath.to_string()).into();
233        Ok(NixString::new_context_from(ctx, outpath.to_string()).into())
234    }
235
236    #[builtin("path")]
237    async fn builtin_path(
238        state: Rc<TvixStoreIO>,
239        co: GenCo,
240        args: Value,
241    ) -> Result<Value, ErrorKind> {
242        let args = args.to_attrs()?;
243
244        let path = match coerce_value_to_path(
245            &co,
246            generators::request_force(&co, args.select_required("path")?.clone()).await,
247        )
248        .await?
249        {
250            Ok(path) => path,
251            Err(cek) => return Ok(cek.into()),
252        };
253
254        let filter = args.select_str("filter");
255
256        // Construct a sha256 hasher, which is needed for flat ingestion.
257        let recursive_ingestion = args
258            .select_str("recursive")
259            .map(|r| r.as_bool())
260            .transpose()?
261            .unwrap_or(true); // Yes, yes, Nix, by default, sets `recursive = true;`.
262
263        let expected_sha256 = args
264            .select_str("sha256")
265            .map(|h| {
266                h.to_str().and_then(|expected| {
267                    match nix_compat::nixhash::from_str(expected.to_str()?, Some("sha256")) {
268                        Ok(NixHash::Sha256(digest)) => Ok(digest),
269                        Ok(_) => unreachable!(),
270                        Err(e) => Err(ErrorKind::InvalidHash(e.to_string())),
271                    }
272                })
273            })
274            .transpose()?;
275
276        import_helper(
277            state,
278            co,
279            path,
280            args.select_str("name"),
281            filter,
282            recursive_ingestion,
283            expected_sha256,
284        )
285        .await
286    }
287
288    #[builtin("filterSource")]
289    async fn builtin_filter_source(
290        state: Rc<TvixStoreIO>,
291        co: GenCo,
292        #[lazy] filter: Value,
293        path: Value,
294    ) -> Result<Value, ErrorKind> {
295        let path =
296            match coerce_value_to_path(&co, generators::request_force(&co, path).await).await? {
297                Ok(path) => path,
298                Err(cek) => return Ok(cek.into()),
299            };
300
301        import_helper(state, co, path, None, Some(&filter), true, None).await
302    }
303
304    #[builtin("storePath")]
305    async fn builtin_store_path(
306        state: Rc<TvixStoreIO>,
307        co: GenCo,
308        path: Value,
309    ) -> Result<Value, ErrorKind> {
310        let p = match &path {
311            Value::String(s) => Path::new(s.as_bytes().to_os_str()?),
312            Value::Path(p) => p.as_path(),
313            _ => {
314                return Err(ErrorKind::TypeError {
315                    expected: "string or path",
316                    actual: path.type_of(),
317                })
318            }
319        };
320
321        // For this builtin, the path needs to start with an absolute store path.
322        let (store_path, _sub_path) = StorePathRef::from_absolute_path_full(p)
323            .map_err(|_e| ImportError::PathNotAbsoluteOrInvalid(p.to_path_buf()))?;
324
325        if state.path_exists(p)? {
326            Ok(Value::String(NixString::new_context_from(
327                [NixContextElement::Plain(store_path.to_absolute_path())].into(),
328                p.as_os_str().as_encoded_bytes(),
329            )))
330        } else {
331            Err(ErrorKind::IO {
332                path: Some(p.to_path_buf()),
333                error: Rc::new(std::io::ErrorKind::NotFound.into()),
334            })
335        }
336    }
337
338    #[builtin("toFile")]
339    async fn builtin_to_file(co: GenCo, name: Value, content: Value) -> Result<Value, ErrorKind> {
340        if name.is_catchable() {
341            return Ok(name);
342        }
343
344        if content.is_catchable() {
345            return Ok(content);
346        }
347
348        let name = name
349            .to_str()
350            .context("evaluating the `name` parameter of builtins.toFile")?;
351        let content = content
352            .to_contextful_str()
353            .context("evaluating the `content` parameter of builtins.toFile")?;
354
355        if content.iter_ctx_derivation().count() > 0
356            || content.iter_ctx_single_outputs().count() > 0
357        {
358            return Err(ErrorKind::UnexpectedContext);
359        }
360
361        let name_str = std::str::from_utf8(name.as_bytes())?;
362        let mut hasher = sha2::Sha256::new();
363        hasher.update(&content);
364        let ca_hash = CAHash::Text(hasher.finalize().into());
365        let store_path: StorePath<&str> =
366            build_ca_path(name_str, &ca_hash, content.iter_ctx_plain(), false)
367                .map_err(|e| tvix_eval::ErrorKind::TvixError(Rc::new(e)))?;
368
369        let abs_path = store_path.to_absolute_path();
370        let context: NixContext = NixContextElement::Plain(abs_path.clone()).into();
371
372        Ok(Value::from(NixString::new_context_from(context, abs_path)))
373    }
374}
375
376pub use import_builtins::builtins as import_builtins;
tvix_glue/builtins/import.rs

tvix_glue/builtins/
import.rs
