Release: v0.3.1-test8-search - thread-safe opendir/readdir crawler, concurrent queue sync, and zero-copy/tail log improvements

Makefile

@@ -122,7 +122,7 @@ BFPILOT_ARCHIVE_OBJS := $(patsubst %.cpp,build/bfpilot-archive/%.o,$(ARCHIVE_WOR
 BFPILOT_ARCHIVE_OBJS += $(patsubst %.c,build/bfpilot-archive/%.o,$(ARCHIVE_SEVENZ_C_SRCS))
 
 COMMON_CFLAGS := -Os -Wall -Werror -Isrc
-COMMON_CFLAGS += -ffunction-sections -fdata-sections -flto
+COMMON_CFLAGS += -ffunction-sections -fdata-sections
 COMMON_CFLAGS += -DVERSION_TAG=\"$(VERSION_TAG)\"
 COMMON_CFLAGS += -DBUILD_VERSION=\"$(BUILD_VERSION)\"
 COMMON_CFLAGS += -DBFPILOT_SDK_PATH=\"$(PS5_PAYLOAD_SDK)\"
@@ -140,7 +140,7 @@ LAUNCHER_INSTALLER_CFLAGS := $(COMMON_CFLAGS)
 LAUNCHER_INSTALLER_CFLAGS += -DBFPILOT_PAYLOAD_NAME=\"bfpilot-launcher-installer\"
 LAUNCHER_INSTALLER_CFLAGS += -DBFPILOT_BUILD_MODE=\"launcher-installer-force\"
 
-COMMON_LDFLAGS := -Wl,--gc-sections -flto
+COMMON_LDFLAGS := -Wl,--gc-sections
 PRIVILEGED_APPINST_LDLIBS := -lkernel_sys -lSceSystemService
 PRIVILEGED_APPINST_LDLIBS += -lSceUserService -lSceAppInstUtil
 ARCHIVE_WORKER_DEFINES := -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_UNIX
@@ -158,8 +158,8 @@ ARCHIVE_WORKER_CXXFLAGS += -Wno-missing-braces -Wno-nontrivial-memcall
 ARCHIVE_WORKER_CFLAGS := -O3 -Wall -Wno-unused-parameter
 ARCHIVE_WORKER_LDFLAGS := -pthread -lSceNotification
 
-CC_CMD := "$(CC)"
-CXX_CMD := "$(CXX)"
+CC_CMD := $(PYTHON) compile_wrapper.py "$(CC)"
+CXX_CMD := $(PYTHON) compile_wrapper.py "$(CXX)"
 STRIP_CMD := "$(STRIP)"
 LD_CMD := "$(LD)"
 DEPLOY_CMD := "$(PS5_DEPLOY)"
@@ -172,8 +172,9 @@ LLVM_BINDIR_POSIX := $(shell cygpath -u "$(LLVM_BINDIR)" 2>/dev/null || printf '
 PS5_PAYLOAD_SDK_SHELL := $(subst \,/,$(PS5_PAYLOAD_SDK))
 LLVM_CONFIG_SHELL := $(subst \,/,$(LLVM_CONFIG))
 LLVM_BINDIR_SHELL := $(subst \,/,$(LLVM_BINDIR))
-RUN_ENV := cd "$(CURDIR_POSIX)" && export LLVM_CONFIG="$(LLVM_CONFIG_SHELL)" && export LLVM_BINDIR="$(LLVM_BINDIR_SHELL)"
+RUN_ENV := cd "$(CURDIR_POSIX)" && export LLVM_CONFIG="$(LLVM_CONFIG_SHELL)" && export LLVM_BINDIR="$(LLVM_BINDIR_SHELL)" && export PATH="$(LLVM_BINDIR_SHELL):$$PATH"
 STRIP_CMD := "$(LLVM_BINDIR_SHELL)/llvm-strip"
+LD_CMD := "$(PS5_PAYLOAD_SDK_SHELL)/win/prospero-lld.exe"
 WINDOWS_LINK_PREFIX := --gc-sections --sysroot="$(PS5_PAYLOAD_SDK_SHELL)"
 WINDOWS_LINK_PREFIX += -L"$(PS5_PAYLOAD_SDK_SHELL)/target/lib"
 WINDOWS_LINK_PREFIX += -L"$(PS5_PAYLOAD_SDK_SHELL)/target/user/homebrew/lib"
@@ -204,7 +205,7 @@ WINDOWS_CXX_LINK_SUFFIX := -lSceNotification -lkernel_web -lSceLibcInternal
 WINDOWS_CXX_LINK_SUFFIX += -lSceNet -lpthread -lc_stub_weak -lkernel_stub_weak
 WINDOWS_CXX_LINK_SUFFIX += -l:crtend.o -l:crtn.o
 define run
-bash -lc '$(RUN_ENV) && $(1)'
+$(RUN_ENV) && $(1)
 endef
 else
 define run

assets/files.html

@@ -101,7 +101,7 @@
       margin: 0 auto;
       padding: 10px 14px;
       display: grid;
-      grid-template-columns: auto minmax(0, 1fr) auto auto;
+      grid-template-columns: auto minmax(0, 1fr) auto auto auto;
       gap: 12px;
       align-items: center;
     }
@@ -628,10 +628,21 @@
 
     @media (max-width: 700px) {
       .top-inner {
-        grid-template-columns: auto minmax(0, 1fr);
+        grid-template-columns: auto minmax(0, 1fr) auto auto;
+      }
+
+      #view-logs {
+        grid-row: 1;
+        grid-column: 3;
+      }
+
+      #terminate-top {
+        grid-row: 1;
+        grid-column: 4;
       }
 
       .status {
+        grid-row: 2;
         grid-column: 1 / -1;
       }
 
@@ -716,6 +727,7 @@
         </div>
       </div>
       <div class="status" id="status"><span class="dot"></span><span>Connecting</span></div>
+      <button id="view-logs" class="button" title="View System Logs">Logs</button>
       <button id="terminate-top" class="danger" title="Terminate Payload">Exit</button>
     </div>
   </header>
@@ -800,6 +812,18 @@
   <input id="folder-picker" class="hidden" type="file" multiple webkitdirectory directory>
   <div id="toast" class="toast"></div>
 
+  <div id="log-overlay" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.8); z-index: 9999; padding: 20px; box-sizing: border-box; flex-direction: column;">
+    <div style="display: flex; justify-content: space-between; align-items: center; background: #222; padding: 10px; border-radius: 8px 8px 0 0;">
+      <h3 style="margin: 0; color: #fff;">System Logs</h3>
+      <div>
+        <button id="log-tab-vsh" class="button primary">PS5 System</button>
+        <button id="log-tab-payload" class="button">Payload</button>
+        <button id="log-close" class="button danger">Close</button>
+      </div>
+    </div>
+    <div id="log-content" style="flex: 1; background: #111; color: #0f0; font-family: monospace; padding: 10px; overflow-y: auto; white-space: pre-wrap; border-radius: 0 0 8px 8px;"></div>
+  </div>
+
   <script>
     const state = {
       path: '/data',
@@ -828,9 +852,7 @@
       }
     };
 
-    function el(id) {
-      return document.getElementById(id);
-    }
+    const el = id => document.getElementById(id);
 
     function cleanPath(path) {
       if (!path || path[0] !== '/') return '/';
@@ -2084,6 +2106,48 @@
       renderPlaces();
       await refreshPlaces(false);
 
+      // Log viewer setup
+      let logTimer = null;
+      let currentLogPath = '/system_data/priv/vshlog/vshlog.0.txt';
+
+      function fetchLogs() {
+        fetch('/fs' + currentLogPath + '?tail=100000').then(r => r.text()).then(t => {
+          const lc = el('log-content');
+          const autoScroll = (lc.scrollHeight - lc.scrollTop <= lc.clientHeight + 50);
+          lc.textContent = t || 'No logs found.';
+          if (autoScroll) lc.scrollTop = lc.scrollHeight;
+        }).catch(e => {
+          const lc = el('log-content');
+          if (lc) lc.textContent = 'Error loading logs: ' + e;
+        });
+      }
+
+      el('view-logs').addEventListener('click', () => {
+        el('log-overlay').style.display = 'flex';
+        fetchLogs();
+        if (logTimer) clearInterval(logTimer);
+        logTimer = setInterval(fetchLogs, 2000);
+      });
+
+      el('log-close').addEventListener('click', () => {
+        el('log-overlay').style.display = 'none';
+        if (logTimer) { clearInterval(logTimer); logTimer = null; }
+      });
+
+      el('log-tab-vsh').addEventListener('click', () => {
+        currentLogPath = '/system_data/priv/vshlog/vshlog.0.txt';
+        el('log-tab-vsh').classList.add('primary');
+        el('log-tab-payload').classList.remove('primary');
+        fetchLogs();
+      });
+
+      el('log-tab-payload').addEventListener('click', () => {
+        currentLogPath = '/data/gemBFPILOT/log.txt';
+        el('log-tab-payload').classList.add('primary');
+        el('log-tab-vsh').classList.remove('primary');
+        fetchLogs();
+      });
+
       el('up').addEventListener('click', () => openPath(parentPath(state.path)));
       el('go').addEventListener('click', refresh);
       el('refresh').addEventListener('click', refresh);

build.cmd

@@ -1,4 +1,4 @@
 @echo off
 set PS5_PAYLOAD_SDK=C:\Users\Blurf\ps5dev\toolchains\ps5-payload-sdk
 set PATH=%PS5_PAYLOAD_SDK%\bin;%PATH%
-make CC=ps5-clang.cmd CXX=ps5-c++.cmd
+make CC=prospero-clang CXX=prospero-clang++

compile_wrapper.py

@@ -0,0 +1,56 @@
+import sys
+import subprocess
+import os
+
+def main():
+    if len(sys.argv) < 2:
+        return
+    
+    compiler_script = sys.argv[1] # C:/Users/.../prospero-clang
+    args = sys.argv[2:]
+    
+    is_cxx = "clang++" in compiler_script
+    
+    sdk = "C:/Users/Blurf/ps5dev/toolchains/ps5-payload-sdk"
+    clang_exe = r"C:\Users\Blurf\scoop\apps\llvm\current\bin\clang.exe"
+    if is_cxx:
+        clang_exe = r"C:\Users\Blurf\scoop\apps\llvm\current\bin\clang++.exe"
+    
+    if not os.path.exists(clang_exe):
+        clang_exe = "clang++" if is_cxx else "clang"
+        
+    cmd = [clang_exe]
+    
+    cmd += ["-target", "x86_64-sie-ps5"]
+    cmd += ["-fvisibility-nodllstorageclass=default"]
+    cmd += ["-isysroot", sdk]
+    cmd += ["--sysroot", sdk]
+
+    if is_cxx:
+        cmd += ["-stdlib++-isystem", f"{sdk}/target/include/c++/v1"]
+        cmd += ["-frtti", "-fexceptions"]
+    cmd += ["-fno-stack-protector", "-fno-plt", "-femulated-tls"]
+    
+    new_args = []
+    for arg in args:
+        if arg.startswith("-DVERSION_TAG=") or arg.startswith("-DBUILD_VERSION=") or arg.startswith("-DBFPILOT_PAYLOAD_NAME=") or arg.startswith("-DBFPILOT_BUILD_MODE="):
+            val = arg.split('=', 1)[1]
+            if val.startswith('\\"') and val.endswith('\\"'):
+                val = '"' + val[2:-2] + '"'
+            elif val.startswith('"') and val.endswith('"'):
+                val = '"' + val[1:-1] + '"'
+            else:
+                val = '"' + val + '"'
+            
+            key = arg.split('=', 1)[0]
+            new_args.append(f"{key}={val}")
+        else:
+            new_args.append(arg)
+            
+    cmd.extend(new_args)
+    
+    res = subprocess.run(cmd)
+    sys.exit(res.returncode)
+
+if __name__ == "__main__":
+    main()

src/archive_worker.cpp

@@ -24,9 +24,12 @@
 #include <dirent.h>
 #include <fcntl.h>
 #include <pthread.h>
+#include <pthread_np.h>
 #include <memory>
 #include <string>
+#include <sys/cpuset.h>
 #include <sys/file.h>
+#include <sys/param.h>
 #include <sys/resource.h>
 #include <sys/stat.h>
 #include <sys/statvfs.h>
@@ -2097,8 +2100,19 @@ bfpilot_archive_start_daemon(void) {
     LogLine("archive daemon thread creation failed errno=%d", errno);
     return -errno;
   }
+
+  // Pin archive extraction to PS5 background core (Core 6)
+  cpuset_t mask;
+  CPU_ZERO(&mask);
+  CPU_SET(6, &mask);
+  int aff_rc = pthread_setaffinity_np(thread, sizeof(cpuset_t), &mask);
+  if (aff_rc != 0) {
+    LogLine("archive daemon setaffinity failed rc=%d", aff_rc);
+  } else {
+    LogLine("archive daemon thread started and pinned to core 6");
+  }
+
   pthread_detach(thread);
-  LogLine("archive daemon thread started");
   return 0;
 }
 

src/fs.c

@@ -15,8 +15,6 @@
 
 #include <sys/stat.h>
 #include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/uio.h>
 
 #include "fs.h"
 #include "mime.h"
@@ -262,34 +260,58 @@ file_request(const http_request_t *req, const char *path) {
     return websrv_send_error_json(req->fd, 404, strerror(errno));
   }
 
+  size_t file_size = (size_t)st.st_size;
+  off_t offset = 0;
+  size_t content_size = file_size;
+
+  /* Support ?tail=N to serve only the last N bytes of the file (for log viewing) */
+  char tail_str[32];
+  if(websrv_get_query_arg(req, "tail", tail_str, sizeof(tail_str))) {
+    size_t tail_bytes = (size_t)strtoull(tail_str, NULL, 10);
+    if(tail_bytes > 0 && tail_bytes < file_size) {
+      offset = (off_t)(file_size - tail_bytes);
+      content_size = tail_bytes;
+    }
+  }
+
+  /* Seek to offset if tailing */
+  if(offset > 0 && lseek(fd, offset, SEEK_SET) < 0) {
+    close(fd);
+    return websrv_send_error_json(req->fd, 500, strerror(errno));
+  }
+
   const char *mime = mime_get_type(path);
   if(websrv_send_headers(req->fd, 200, mime ? mime : "application/octet-stream",
-                         (size_t)st.st_size, NULL) != 0) {
+                         content_size, NULL) != 0) {
+    close(fd);
+    return -1;
+  }
+
+  char *buf = malloc(STREAM_BUF_SIZE);
+  if(!buf) {
     close(fd);
     return -1;
   }
 
   int rc = 0;
-  off_t offset = 0;
-  size_t remaining = (size_t)st.st_size;
-
+  size_t remaining = content_size;
   while(remaining > 0) {
-    off_t sent = 0;
-    if(sendfile(fd, req->fd, offset, remaining, NULL, &sent, 0) < 0) {
-      if(errno == EINTR || errno == EAGAIN) {
-        if(sent > 0) {
-          offset += sent;
-          remaining -= sent;
-        }
-        continue;
-      }
+    size_t to_read = remaining < STREAM_BUF_SIZE ? remaining : STREAM_BUF_SIZE;
+    ssize_t n = read(fd, buf, to_read);
+    if(n < 0) {
+      if(errno == EINTR) continue;
       rc = -1;
       break;
     }
-    if(sent == 0) break;
-    offset += sent;
-    remaining -= sent;
+    if(n == 0) break;
+    if(websrv_write_all(req->fd, buf, (size_t)n) != 0) {
+      rc = -1;
+      break;
+    }
+    remaining -= (size_t)n;
   }
+
+  free(buf);
   close(fd);
   return rc;
 }

src/search.c

@@ -271,7 +271,6 @@ search_arena_alloc(string_arena_t **arena, size_t size) {
     if(!new_a) return NULL;
     new_a->ptr = malloc(cap);
     if(!new_a->ptr) {
-      free(new_a);
       free(new_a);
       return NULL;
     }
@@ -581,7 +580,7 @@ search_save_index(search_index_t *idx) {
 
   idx_header_t hdr = {0};
   snprintf(hdr.magic, sizeof(hdr.magic), "BFPILOT_IDX");
-  hdr.version = 1;
+  hdr.version = 2;
   hdr.count = idx->count;
   hdr.built_at = idx->built_at;
   hdr.build_ms = idx->build_ms;
@@ -628,7 +627,7 @@ search_load_index(void) {
     fclose(f);
     return NULL;
   }
-  if(strcmp(hdr.magic, "BFPILOT_IDX") != 0 || hdr.version != 1) {
+  if(strcmp(hdr.magic, "BFPILOT_IDX") != 0 || hdr.version != 2) {
     fclose(f);
     return NULL;
   }
@@ -846,10 +845,10 @@ static void *crawler_thread_func(void *arg) {
 
   for(;;) {
     pthread_mutex_lock(&q->lock);
-    while(q->head >= q->count && !q->done_pushing && !search_cancelled()) {
+    while(q->head >= q->count && (!q->done_pushing || q->active_workers > 0) && !search_cancelled()) {
       pthread_cond_wait(&q->cond, &q->lock);
     }
-    if(search_cancelled() || (q->head >= q->count && q->done_pushing)) {
+    if(search_cancelled() || (q->head >= q->count && q->done_pushing && q->active_workers == 0)) {
       pthread_mutex_unlock(&q->lock);
       break;
     }
@@ -858,13 +857,13 @@ static void *crawler_thread_func(void *arg) {
     q->active_workers++;
     pthread_mutex_unlock(&q->lock);
 
-    int fd = open(dir_path, O_RDONLY | O_DIRECTORY);
-    if(fd < 0) {
+    DIR *dir = opendir(dir_path);
+    if(!dir) {
       pthread_mutex_lock(&shared->index_lock);
       shared->saved_errno = errno;
       idx->errors++;
       pthread_mutex_unlock(&shared->index_lock);
-      search_diag_log("FAIL: open(%s) = %s", dir_path, strerror(errno));
+      search_diag_log("FAIL: opendir(%s) = %s", dir_path, strerror(errno));
       free(dir_path);
 
       pthread_mutex_lock(&q->lock);
@@ -884,124 +883,112 @@ static void *crawler_thread_func(void *arg) {
     int is_truncated = idx->truncated;
     pthread_mutex_unlock(&shared->index_lock);
 
-    char buf[65536];
+    struct dirent *ent;
     int read_errno = 0;
     for(;;) {
-      int nread = getdents(fd, buf, sizeof(buf));
-      if(nread < 0) {
+      errno = 0;
+      ent = readdir(dir);
+      if(!ent) {
         read_errno = errno;
         break;
       }
-      if(nread == 0) break;
 
-      for(int i = 0; i < nread; ) {
-        struct dirent *ent = (struct dirent *)(buf + i);
-        if(ent->d_reclen == 0) break;
-        
-        if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
-          i += ent->d_reclen;
-          continue;
-        }
-        if(search_cancelled()) break;
-
-        char child[1024];
-        if(strlen(dir_path) + strlen(ent->d_name) + 2 >= sizeof(child)) {
-          pthread_mutex_lock(&shared->index_lock);
-          idx->skipped++;
-          pthread_mutex_unlock(&shared->index_lock);
-          i += ent->d_reclen;
-          continue;
-        }
-        join_path(child, sizeof(child), dir_path, ent->d_name);
-
-        if(shared->system_mode && search_skip_system_path(child)) {
-          pthread_mutex_lock(&shared->index_lock);
-          idx->skipped++;
-          pthread_mutex_unlock(&shared->index_lock);
-          i += ent->d_reclen;
-          continue;
-        }
-
-        struct stat st = {0};
-        int stat_rc = 0;
-        if (ent->d_type == DT_REG) {
-          st.st_mode = S_IFREG | 0644;
-          st.st_size = 0;
-          st.st_mtime = 0;
-        } else {
-          stat_rc = lstat(child, &st);
-        }
-
-        if(stat_rc != 0) {
-          pthread_mutex_lock(&shared->index_lock);
-          idx->errors++;
-          pthread_mutex_unlock(&shared->index_lock);
-          i += ent->d_reclen;
-          continue;
-        }
+      if(!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) {
+        continue;
+      }
+      if(search_cancelled()) break;
 
+      char child[1024];
+      if(strlen(dir_path) + strlen(ent->d_name) + 2 >= sizeof(child)) {
         pthread_mutex_lock(&shared->index_lock);
-        int add_rc = search_index_add(idx, child, &st);
-        if(add_rc < 0) {
-          shared->rc = -1;
-          shared->saved_errno = ENOMEM;
-          snprintf(shared->error, sizeof(shared->error), "%s", "out of memory");
-          pthread_mutex_unlock(&shared->index_lock);
-          break;
-        }
+        idx->skipped++;
+        pthread_mutex_unlock(&shared->index_lock);
+        continue;
+      }
+      join_path(child, sizeof(child), dir_path, ent->d_name);
 
-        if(S_ISDIR(st.st_mode)) {
-          if(!idx->truncated) {
-            /* Use dev+ino to detect symlink loops across any device */
-            if(visited_set_add(shared->visited, st.st_dev, st.st_ino) > 0) {
-              char *copy = search_strdup(child);
-              if(!copy) {
+      if(shared->system_mode && search_skip_system_path(child)) {
+        pthread_mutex_lock(&shared->index_lock);
+        idx->skipped++;
+        pthread_mutex_unlock(&shared->index_lock);
+        continue;
+      }
+
+      struct stat st = {0};
+      int stat_rc = 0;
+      if (ent->d_type == DT_REG) {
+        st.st_mode = S_IFREG | 0644;
+        st.st_size = 0;
+        st.st_mtime = 0;
+      } else {
+        stat_rc = lstat(child, &st);
+      }
+
+      if(stat_rc != 0) {
+        pthread_mutex_lock(&shared->index_lock);
+        idx->errors++;
+        pthread_mutex_unlock(&shared->index_lock);
+        continue;
+      }
+
+      pthread_mutex_lock(&shared->index_lock);
+      int add_rc = search_index_add(idx, child, &st);
+      if(add_rc < 0) {
+        shared->rc = -1;
+        shared->saved_errno = ENOMEM;
+        snprintf(shared->error, sizeof(shared->error), "%s", "out of memory");
+        pthread_mutex_unlock(&shared->index_lock);
+        break;
+      }
+
+      if(S_ISDIR(st.st_mode)) {
+        if(!idx->truncated) {
+          /* Use dev+ino to detect symlink loops across any device */
+          if(visited_set_add(shared->visited, st.st_dev, st.st_ino) > 0) {
+            char *copy = search_strdup(child);
+            if(!copy) {
+              shared->rc = -1;
+              shared->saved_errno = ENOMEM;
+              snprintf(shared->error, sizeof(shared->error), "%s", "out of memory");
+              pthread_mutex_unlock(&shared->index_lock);
+              break;
+            }
+            pthread_mutex_lock(&q->lock);
+            if(q->count >= q->cap) {
+              size_t next = q->cap ? q->cap * 2 : 256;
+              char **items = (char **)realloc(q->dirs, next * sizeof(*items));
+              if(items) {
+                q->dirs = items;
+                q->cap = next;
+              } else {
+                free(copy);
                 shared->rc = -1;
                 shared->saved_errno = ENOMEM;
                 snprintf(shared->error, sizeof(shared->error), "%s", "out of memory");
+                pthread_mutex_unlock(&q->lock);
                 pthread_mutex_unlock(&shared->index_lock);
                 break;
               }
-              pthread_mutex_lock(&q->lock);
-              if(q->count >= q->cap) {
-                size_t next = q->cap ? q->cap * 2 : 256;
-                char **items = realloc(q->dirs, next * sizeof(*items));
-                if(items) {
-                  q->dirs = items;
-                  q->cap = next;
-                } else {
-                  free(copy);
-                  shared->rc = -1;
-                  shared->saved_errno = ENOMEM;
-                  snprintf(shared->error, sizeof(shared->error), "%s", "out of memory");
-                  pthread_mutex_unlock(&q->lock);
-                  pthread_mutex_unlock(&shared->index_lock);
-                  break;
-                }
-              }
-              q->dirs[q->count++] = copy;
-              pthread_cond_signal(&q->cond);
-              pthread_mutex_unlock(&q->lock);
-            } else {
-              search_diag_log("SKIP LOOP: %s", child);
             }
+            q->dirs[q->count++] = copy;
+            pthread_cond_signal(&q->cond);
+            pthread_mutex_unlock(&q->lock);
+          } else {
+            search_diag_log("SKIP LOOP: %s", child);
           }
         }
-
-        if((idx->count % BFPILOT_SEARCH_PROGRESS_INTERVAL) == 0) {
-          publish_progress(idx, child, NULL, 0);
-        }
-        is_truncated = idx->truncated;
-        pthread_mutex_unlock(&shared->index_lock);
-
-        if(is_truncated) break;
-        i += ent->d_reclen;
       }
-      
-      if(search_cancelled() || shared->rc != 0 || is_truncated) break;
+
+      if((idx->count % BFPILOT_SEARCH_PROGRESS_INTERVAL) == 0) {
+        publish_progress(idx, child, NULL, 0);
+      }
+      is_truncated = idx->truncated;
+      pthread_mutex_unlock(&shared->index_lock);
+
+      if(is_truncated) break;
     }
 
-    close(fd);
+    closedir(dir);
     free(dir_path);
 
     pthread_mutex_lock(&shared->index_lock);