diff --git a/pkgs/test/nixpkgs-check-by-name/src/check_result.rs b/pkgs/test/nixpkgs-check-by-name/src/check_result.rs
index df894df45c71..79d3dbe67662 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/check_result.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/check_result.rs
@@ -4,7 +4,21 @@ use std::fmt;
 use std::io;
 use std::path::PathBuf;
 
-pub enum CheckError {}
+pub enum CheckError {
+    OutsidePathReference {
+        relative_package_dir: PathBuf,
+        subpath: PathBuf,
+        line: usize,
+        text: String,
+    },
+    UnresolvablePathReference {
+        relative_package_dir: PathBuf,
+        subpath: PathBuf,
+        line: usize,
+        text: String,
+        io_error: io::Error,
+    },
+}
 
 impl CheckError {
     pub fn into_result<A>(self) -> CheckResult<A> {
@@ -14,7 +28,24 @@ impl CheckError {
 
 impl fmt::Display for CheckError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match *self {}
+        match self {
+            CheckError::OutsidePathReference { relative_package_dir, subpath, line, text } =>
+                write!(
+                    f,
+                    "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.",
+                    relative_package_dir.display(),
+                    subpath.display(),
+                    text,
+                ),
+            CheckError::UnresolvablePathReference { relative_package_dir, subpath, line, text, io_error } =>
+                write!(
+                    f,
+                    "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {io_error}.",
+                    relative_package_dir.display(),
+                    subpath.display(),
+                    text,
+                ),
+        }
     }
 }
 
diff --git a/pkgs/test/nixpkgs-check-by-name/src/references.rs b/pkgs/test/nixpkgs-check-by-name/src/references.rs
index 16dc60729c42..30eaee0d7484 100644
--- a/pkgs/test/nixpkgs-check-by-name/src/references.rs
+++ b/pkgs/test/nixpkgs-check-by-name/src/references.rs
@@ -1,3 +1,4 @@
+use crate::check_result::{pass, write_check_result, CheckError};
 use crate::structure::Nixpkgs;
 use crate::utils;
 use crate::utils::{ErrorWriter, LineIndex};
@@ -155,29 +156,34 @@ fn check_nix_file<W: io::Write>(
 
         // Resolves the reference of the Nix path
         // turning `../baz` inside `/foo/bar/default.nix` to `/foo/baz`
-        match parent_dir.join(Path::new(&text)).canonicalize() {
+        let check_result = match parent_dir.join(Path::new(&text)).canonicalize() {
             Ok(target) => {
                 // Then checking if it's still in the package directory
                 // No need to handle the case of it being inside the directory, since we scan through the
                 // entire directory recursively anyways
                 if let Err(_prefix_error) = target.strip_prefix(context.absolute_package_dir) {
-                    context.error_writer.write(&format!(
-                        "{}: File {} at line {line} contains the path expression \"{}\" which may point outside the directory of that package.",
-                        context.relative_package_dir.display(),
-                        subpath.display(),
+                    CheckError::OutsidePathReference {
+                        relative_package_dir: context.relative_package_dir.clone(),
+                        subpath: subpath.to_path_buf(),
+                        line,
                         text,
-                    ))?;
+                    }
+                    .into_result()
+                } else {
+                    pass(())
                 }
             }
-            Err(e) => {
-                context.error_writer.write(&format!(
-                    "{}: File {} at line {line} contains the path expression \"{}\" which cannot be resolved: {e}.",
-                    context.relative_package_dir.display(),
-                    subpath.display(),
-                    text,
-                ))?;
+            Err(e) => CheckError::UnresolvablePathReference {
+                relative_package_dir: context.relative_package_dir.clone(),
+                subpath: subpath.to_path_buf(),
+                line,
+                text,
+                io_error: e,
             }
+            .into_result(),
         };
+
+        write_check_result(context.error_writer, check_result)?;
     }
 
     Ok(())