Merge Organize on SMB, FTP, SFTP and WebDAV servers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
phantomptrandClaude Opus 5.5 committed 2026-10-18 22:52:57 -07:00
commit 744f34ebbb
39 files changed
+2685 -337

No files matched your search

+10 -2
View File
@@ -1160,8 +1160,9 @@ Yes. Save the server under **Connections** (SMB, FTP, FTPS or SFTP), then in **C
Folders** press **Add a folder on <server>** and pick the folder. It is scanned where it is:
packages, game folders and covers are read over the network, and nothing is copied to this
computer. Its games install straight from the server, and **Convert…** reads them in place.
Move to Trash, Organize, junk clean-up and Copy install link work on this computer's folders
only. A mounted share also still works as an ordinary folder.
**Organize packages** works there too, renaming on the server itself; Move to Trash, junk
clean-up and Copy install link work on this computer's folders only. A mounted share also still
works as an ordinary folder.
**Q: Does it keep the computer awake or rescan all the time?**
No. **Look for changes automatically** (in Folders) rescans only while the app is open, never
@@ -1198,6 +1199,13 @@ a second copy of the same release is filed beside it and marked `DUPLICATE`, spl
(`_0`, `_1`, …) move together, and a package whose kind had to be inferred is marked `_UNSURE`.
You see every move first and can untick any. Nothing is overwritten or deleted and files are
only renamed, never copied. Every run can be undone from **Earlier runs** in the same window.
A folder on a saved server (SMB, FTP, SFTP, WebDAV) is organized on the server: each move is a
rename there, a package never moves to another server or folder, and a connection that drops
stops the run where it is, with Undo still putting back whatever moved. On WebDAV a folder the
run emptied is left in place (empty), and the result lists it: a WebDAV delete of a folder takes
everything in it, so ps5upload never deletes one there. FTP has no rename that
refuses a taken name, so there the name is checked first; a file another program writes to that
exact name in the instant between the check and the rename could be replaced.
**Q: What are the junk files, and is cleaning them safe?**
A Mac writes a hidden `._` file beside every file it copies to an exFAT drive (each one takes a
+5 -1
View File
@@ -207,6 +207,8 @@ export interface OrganizeSkipped {
export interface OrganizePlan {
token: string;
containers: string[];
/** How to show each container: its path, or "Server: /path" on a saved server. */
labels?: Record<string, string>;
moves: OrganizeMove[];
skipped: OrganizeSkipped[];
unsure_count: number;
@@ -224,6 +226,8 @@ export interface OrganizeResult {
export interface OrganizeRun {
id: string;
container_root: string;
/** The folder as shown: its path, or "Server: /path" on a saved server. */
label?: string;
created_at: string;
moved: number;
failed: number;
@@ -332,7 +336,7 @@ export const collection = {
}),
organizeRuns: () => call<OrganizeRun[]>("/organize/runs"),
organizeRevert: (id: string) =>
call<{ reverted: OrganizeMove[]; problems: unknown[] }>(
call<{ reverted: OrganizeMove[]; problems: unknown[]; note?: string }>(
"/organize/revert",
{ method: "POST", body: JSON.stringify({ id }) },
),
+1 -1
View File
@@ -3334,7 +3334,7 @@ upload_as_image_title: "أو أرسلها كصورة لعبة واحدة",
"collection.add_server_folder": "إضافة مجلد على {name}",
"collection.no_servers_hint": "ألعاب على NAS؟ احفظه في الاتصالات (SMB أو FTP أو SFTP/WebDAV)، وتُضاف مجلداته هنا دون تركيبها.",
"collection.pick_server_folder": "المجلد على {name} الذي يحوي ألعابك",
"collection.server_folders_hint": "يُقرأ المجلد على خادم محفوظ (SMB أو FTP أو SFTP/WebDAV) في مكانه: لا يُنسخ شيء إلى هذا الحاسوب، وتُثبَّت ألعابه مباشرة من الخادم. النقل إلى سلة المهملات والتنظيم يعملان على مجلدات هذا الحاسوب فقط.",
"collection.server_folders_hint": "يُقرأ المجلد على خادم محفوظ (SMB أو FTP أو SFTP/WebDAV) في مكانه: لا يُنسخ شيء إلى هذا الحاسوب، وتُثبَّت ألعابه مباشرة من الخادم. التنظيم يعمل هناك أيضًا بإعادة التسمية على الخادم نفسه؛ أما النقل إلى سلة المهملات فيعمل على مجلدات هذا الحاسوب فقط.",
bottleneck_source_v2: "قراءة المصدر",
android_local_network_allow: "السماح",
android_local_network_body: "يمنع Android التطبيق من الوصول إلى جهاز PS5 حتى تسمح بذلك.",
+1 -1
View File
@@ -3334,7 +3334,7 @@ upload_as_image_title: "অথবা একটি গেম ইমেজ হি
"collection.add_server_folder": "{name}-এ একটি ফোল্ডার যোগ করুন",
"collection.no_servers_hint": "গেম NAS-এ? সংযোগে (SMB, FTP বা SFTP/WebDAV) সেটি সংরক্ষণ করুন, তারপর মাউন্ট না করেই এর ফোল্ডার এখানে যোগ করা যাবে।",
"collection.pick_server_folder": "{name}-এ আপনার গেম থাকা ফোল্ডার",
"collection.server_folders_hint": "সংরক্ষিত সার্ভারের (SMB, FTP, SFTP/WebDAV) ফোল্ডার জায়গাতেই পড়া হয়: এই কম্পিউটারে কিছু কপি হয় না, আর এর গেম সরাসরি সার্ভার থেকে ইনস্টল হয়। ট্র্যাশে সরান ও গোছান শুধু এই কম্পিউটারের ফোল্ডারে কাজ করে।",
"collection.server_folders_hint": "সংরক্ষিত সার্ভারের (SMB, FTP, SFTP/WebDAV) ফোল্ডার জায়গাতেই পড়া হয়: এই কম্পিউটারে কিছু কপি হয় না, আর এর গেম সরাসরি সার্ভার থেকে ইনস্টল হয়। গোছান সেখানেও কাজ করে, সার্ভারেই নাম বদলে; ট্র্যাশে সরান শুধু এই কম্পিউটারের ফোল্ডারে কাজ করে।",
bottleneck_source_v2: "উৎস পড়া",
android_local_network_allow: "অনুমতি দিন",
android_local_network_body: "আপনি অনুমতি না দেওয়া পর্যন্ত Android অ্যাপটিকে আপনার PS5-এ পৌঁছাতে দেয় না।",
+1 -1
View File
@@ -3342,7 +3342,7 @@ upload_as_image_title: "Oder als ein Spiel-Image senden",
"collection.add_server_folder": "Ordner auf {name} hinzufügen",
"collection.no_servers_hint": "Spiele auf einem NAS? Speichere es unter Verbindungen (SMB, FTP oder SFTP/WebDAV), dann lassen sich seine Ordner hier hinzufügen, ohne sie einzubinden.",
"collection.pick_server_folder": "Der Ordner auf {name} mit deinen Spielen",
"collection.server_folders_hint": "Ein Ordner auf einem gespeicherten Server (SMB, FTP, SFTP/WebDAV) wird an Ort und Stelle gelesen: Nichts wird auf diesen Computer kopiert, und seine Spiele installieren direkt vom Server. In den Papierkorb und Aufräumen gelten nur für Ordner auf diesem Computer.",
"collection.server_folders_hint": "Ein Ordner auf einem gespeicherten Server (SMB, FTP, SFTP/WebDAV) wird an Ort und Stelle gelesen: Nichts wird auf diesen Computer kopiert, und seine Spiele installieren direkt vom Server. Aufräumen funktioniert dort ebenfalls und benennt direkt auf dem Server um; In den Papierkorb gilt nur für Ordner auf diesem Computer.",
bottleneck_source_v2: "Lesen der Quelle",
android_local_network_allow: "Zulassen",
android_local_network_body: "Android verhindert, dass die App deine PS5 erreicht, bis du es zulässt.",
+1 -1
View File
@@ -4006,7 +4006,7 @@ upload_as_image_title: "Or send it as one game image",
"collection.add_server_folder": "Add a folder on {name}",
"collection.no_servers_hint": "Games on a NAS? Save it under Connections (SMB, FTP or SFTP/WebDAV), and its folders can be added here without mounting them.",
"collection.pick_server_folder": "The folder on {name} that holds your games",
"collection.server_folders_hint": "A folder on a saved server (SMB, FTP, SFTP/WebDAV) is read in place: nothing is copied to this computer, and its games install straight from the server. Move to Trash and Organize work on this computer's folders only.",
"collection.server_folders_hint": "A folder on a saved server (SMB, FTP, SFTP/WebDAV) is read in place: nothing is copied to this computer, and its games install straight from the server. Organize works there too, renaming on the server itself; Move to Trash works on this computer's folders only.",
android_local_network_allow: "Allow",
android_local_network_body: "Android blocks the app from reaching your PS5 until you allow it.",
android_local_network_title: "Allow PS5Upload on your local network",
+1 -1
View File
@@ -3284,7 +3284,7 @@ upload_as_image_title: "O envíalo como una sola imagen de juego",
"collection.add_server_folder": "Añadir una carpeta de {name}",
"collection.no_servers_hint": "¿Juegos en un NAS? Guárdalo en Conexiones (SMB, FTP o SFTP/WebDAV) y podrás añadir aquí sus carpetas sin montarlas.",
"collection.pick_server_folder": "La carpeta de {name} que contiene tus juegos",
"collection.server_folders_hint": "Una carpeta en un servidor guardado (SMB, FTP, SFTP/WebDAV) se lee donde está: no se copia nada a este ordenador y sus juegos se instalan directamente desde el servidor. Mover a la Papelera y Organizar solo funcionan con carpetas de este ordenador.",
"collection.server_folders_hint": "Una carpeta en un servidor guardado (SMB, FTP, SFTP/WebDAV) se lee donde está: no se copia nada a este ordenador y sus juegos se instalan directamente desde el servidor. Organizar también funciona allí, renombrando en el propio servidor; Mover a la Papelera solo funciona con carpetas de este ordenador.",
bottleneck_source_v2: "la lectura del origen",
android_local_network_allow: "Permitir",
android_local_network_body: "Android impide que la app llegue a tu PS5 hasta que lo permitas.",
+1 -1
View File
@@ -3666,7 +3666,7 @@ upload_as_image_title: "یا آن را به صورت یک ایمیج بازی ب
"collection.add_server_folder": "افزودن پوشه‌ای روی {name}",
"collection.no_servers_hint": "بازی‌ها روی NAS هستند؟ آن را در اتصال‌ها (SMB، FTP یا SFTP/WebDAV) ذخیره کنید تا پوشه‌هایش بدون مونت کردن اینجا اضافه شوند.",
"collection.pick_server_folder": "پوشه‌ای روی {name} که بازی‌هایتان در آن است",
"collection.server_folders_hint": "پوشه روی سرور ذخیره‌شده (SMB، FTP، SFTP/WebDAV) در همان جا خوانده می‌شود: چیزی به این رایانه کپی نمی‌شود و بازی‌هایش مستقیم از سرور نصب می‌شوند. انتقال به سطل زباله و مرتب‌سازی فقط روی پوشه‌های این رایانه کار می‌کنند.",
"collection.server_folders_hint": "پوشه روی سرور ذخیره‌شده (SMB، FTP، SFTP/WebDAV) در همان جا خوانده می‌شود: چیزی به این رایانه کپی نمی‌شود و بازی‌هایش مستقیم از سرور نصب می‌شوند. مرتب‌سازی آنجا هم کار می‌کند و نام‌ها را روی خود سرور تغییر می‌دهد؛ انتقال به سطل زباله فقط روی پوشه‌های این رایانه کار می‌کند.",
bottleneck_source_v2: "خواندن منبع",
android_local_network_allow: "اجازه دادن",
android_local_network_body: "تا وقتی اجازه ندهید، Android جلوی دسترسی برنامه به PS5 را می‌گیرد.",
+1 -1
View File
@@ -3352,7 +3352,7 @@ upload_as_image_title: "Ou l’envoyer en une seule image de jeu",
"collection.add_server_folder": "Ajouter un dossier de {name}",
"collection.no_servers_hint": "Des jeux sur un NAS ? Enregistrez-le dans Connexions (SMB, FTP ou SFTP/WebDAV) et ses dossiers pourront être ajoutés ici sans les monter.",
"collection.pick_server_folder": "Le dossier de {name} qui contient vos jeux",
"collection.server_folders_hint": "Un dossier sur un serveur enregistré (SMB, FTP, SFTP/WebDAV) est lu sur place : rien n’est copié sur cet ordinateur et ses jeux s’installent directement depuis le serveur. Mettre à la Corbeille et Ranger ne concernent que les dossiers de cet ordinateur.",
"collection.server_folders_hint": "Un dossier sur un serveur enregistré (SMB, FTP, SFTP/WebDAV) est lu sur place : rien n’est copié sur cet ordinateur et ses jeux s’installent directement depuis le serveur. Ranger fonctionne aussi, en renommant sur le serveur lui-même ; Mettre à la Corbeille ne concerne que les dossiers de cet ordinateur.",
bottleneck_source_v2: "la lecture de la source",
android_local_network_allow: "Autoriser",
android_local_network_body: "Android empêche l’app d’atteindre ta PS5 tant que tu ne l’autorises pas.",
+1 -1
View File
@@ -3350,7 +3350,7 @@ upload_as_image_title: "या इसे एक गेम इमेज के
"collection.add_server_folder": "{name} पर फ़ोल्डर जोड़ें",
"collection.no_servers_hint": "गेम NAS पर हैं? इसे कनेक्शन में सहेजें (SMB, FTP या SFTP/WebDAV), फिर इसके फ़ोल्डर बिना माउंट किए यहाँ जोड़े जा सकते हैं।",
"collection.pick_server_folder": "{name} पर आपके गेम वाला फ़ोल्डर",
"collection.server_folders_hint": "सहेजे गए सर्वर (SMB, FTP, SFTP/WebDAV) का फ़ोल्डर वहीं से पढ़ा जाता है: इस कंप्यूटर पर कुछ कॉपी नहीं होता, और उसके गेम सीधे सर्वर से इंस्टॉल होते हैं। ट्रैश में भेजें और व्यवस्थित करें केवल इस कंप्यूटर के फ़ोल्डरों पर काम करते हैं।",
"collection.server_folders_hint": "सहेजे गए सर्वर (SMB, FTP, SFTP/WebDAV) का फ़ोल्डर वहीं से पढ़ा जाता है: इस कंप्यूटर पर कुछ कॉपी नहीं होता, और उसके गेम सीधे सर्वर से इंस्टॉल होते हैं। व्यवस्थित करें वहाँ भी काम करता है, सर्वर पर ही नाम बदलकर; ट्रैश में भेजें केवल इस कंप्यूटर के फ़ोल्डरों पर काम करता है।",
bottleneck_source_v2: "स्रोत पढ़ना",
android_local_network_allow: "अनुमति दें",
android_local_network_body: "जब तक आप अनुमति नहीं देते, Android ऐप को आपके PS5 तक पहुँचने से रोकता है।",
+1 -1
View File
@@ -3649,7 +3649,7 @@ upload_as_image_title: "Vagy küldd el egyetlen játékképfájlként",
"collection.add_server_folder": "Mappa hozzáadása innen: {name}",
"collection.no_servers_hint": "Játékok egy NAS-on? Mentsd el a Kapcsolatok között (SMB, FTP vagy SFTP/WebDAV), és a mappái csatolás nélkül hozzáadhatók itt.",
"collection.pick_server_folder": "A játékaidat tartalmazó mappa itt: {name}",
"collection.server_folders_hint": "Egy mentett szerveren (SMB, FTP, SFTP/WebDAV) lévő mappát a helyén olvasunk: semmi sem másolódik erre a számítógépre, a játékai pedig közvetlenül a szerverről települnek. A Kukába helyezés és a Rendezés csak ennek a számítógépnek a mappáira vonatkozik.",
"collection.server_folders_hint": "Egy mentett szerveren (SMB, FTP, SFTP/WebDAV) lévő mappát a helyén olvasunk: semmi sem másolódik erre a számítógépre, a játékai pedig közvetlenül a szerverről települnek. A Rendezés ott is működik, magán a szerveren nevez át; a Kukába helyezés csak ennek a számítógépnek a mappáira vonatkozik.",
bottleneck_source_v2: "a forrás olvasása",
android_local_network_allow: "Engedélyezés",
android_local_network_body: "Amíg nem engedélyezed, az Android nem engedi, hogy az alkalmazás elérje a PS5-öt.",
+1 -1
View File
@@ -3343,7 +3343,7 @@ upload_as_image_title: "Atau kirim sebagai satu image game",
"collection.add_server_folder": "Tambah folder di {name}",
"collection.no_servers_hint": "Game di NAS? Simpan di Koneksi (SMB, FTP, atau SFTP/WebDAV), lalu foldernya bisa ditambahkan di sini tanpa dipasang.",
"collection.pick_server_folder": "Folder di {name} yang berisi game kamu",
"collection.server_folders_hint": "Folder di server tersimpan (SMB, FTP, SFTP/WebDAV) dibaca di tempatnya: tidak ada yang disalin ke komputer ini, dan game-nya terpasang langsung dari server. Pindahkan ke Tempat Sampah dan Rapikan hanya berlaku untuk folder di komputer ini.",
"collection.server_folders_hint": "Folder di server tersimpan (SMB, FTP, SFTP/WebDAV) dibaca di tempatnya: tidak ada yang disalin ke komputer ini, dan game-nya terpasang langsung dari server. Rapikan juga berfungsi di sana, dengan mengganti nama langsung di server; Pindahkan ke Tempat Sampah hanya berlaku untuk folder di komputer ini.",
bottleneck_source_v2: "membaca sumber",
android_local_network_allow: "Izinkan",
android_local_network_body: "Android memblokir aplikasi agar tidak bisa menjangkau PS5 sampai kamu mengizinkannya.",
+1 -1
View File
@@ -3284,7 +3284,7 @@ upload_as_image_title: "Oppure invialo come un’unica immagine di gioco",
"collection.add_server_folder": "Aggiungi una cartella su {name}",
"collection.no_servers_hint": "Giochi su un NAS? Salvalo in Connessioni (SMB, FTP o SFTP/WebDAV) e le sue cartelle si potranno aggiungere qui senza montarle.",
"collection.pick_server_folder": "La cartella su {name} che contiene i tuoi giochi",
"collection.server_folders_hint": "Una cartella su un server salvato (SMB, FTP, SFTP/WebDAV) viene letta dov’è: non viene copiato nulla su questo computer e i suoi giochi si installano direttamente dal server. Sposta nel Cestino e Riordina valgono solo per le cartelle di questo computer.",
"collection.server_folders_hint": "Una cartella su un server salvato (SMB, FTP, SFTP/WebDAV) viene letta dov’è: non viene copiato nulla su questo computer e i suoi giochi si installano direttamente dal server. Anche Riordina funziona lì, rinominando sul server stesso; Sposta nel Cestino vale solo per le cartelle di questo computer.",
bottleneck_source_v2: "la lettura della sorgente",
android_local_network_allow: "Consenti",
android_local_network_body: "Android impedisce all’app di raggiungere la tua PS5 finché non lo consenti.",
+1 -1
View File
@@ -3355,7 +3355,7 @@ upload_as_image_title: "または 1 つのゲームイメージとして送る",
"collection.add_server_folder": "{name} のフォルダーを追加",
"collection.no_servers_hint": "NAS にゲームがありますか? 接続 (SMB、FTP、SFTP/WebDAV) に保存すると、マウントせずにそのフォルダーをここに追加できます。",
"collection.pick_server_folder": "{name} 上のゲームが入っているフォルダー",
"collection.server_folders_hint": "保存したサーバー (SMB、FTP、SFTP/WebDAV) のフォルダーはその場で読み取ります。このコンピューターには何もコピーされず、ゲームはサーバーから直接インストールされます。ゴミ箱に移すと整理は、このコンピューターのフォルダーだけが対象です。",
"collection.server_folders_hint": "保存したサーバー (SMB、FTP、SFTP/WebDAV) のフォルダーはその場で読み取ります。このコンピューターには何もコピーされず、ゲームはサーバーから直接インストールされます。整理はサーバー上でも使え、サーバー上で直接名前を変更します。ゴミ箱に移すは、このコンピューターのフォルダーだけが対象です。",
bottleneck_source_v2: "ソースの読み取り",
android_local_network_allow: "許可",
android_local_network_body: "許可するまで、Android はこのアプリが PS5 に接続するのをブロックします。",
+1 -1
View File
@@ -3353,7 +3353,7 @@ upload_as_image_title: "또는 게임 이미지 하나로 보내기",
"collection.add_server_folder": "{name}의 폴더 추가",
"collection.no_servers_hint": "NAS에 게임이 있나요? 연결(SMB, FTP, SFTP/WebDAV)에 저장하면 마운트하지 않고도 그 폴더를 여기에 추가할 수 있습니다.",
"collection.pick_server_folder": "게임이 들어 있는 {name}의 폴더",
"collection.server_folders_hint": "저장된 서버(SMB, FTP, SFTP/WebDAV)의 폴더는 그 자리에서 읽습니다. 이 컴퓨터로 아무것도 복사하지 않고, 게임은 서버에서 바로 설치됩니다. 휴지통으로 이동과 정리는 이 컴퓨터의 폴더에만 적용됩니다.",
"collection.server_folders_hint": "저장된 서버(SMB, FTP, SFTP/WebDAV)의 폴더는 그 자리에서 읽습니다. 이 컴퓨터로 아무것도 복사하지 않고, 게임은 서버에서 바로 설치됩니다. 정리는 서버에서도 작동하며 서버에서 직접 이름을 바꿉니다. 휴지통으로 이동은 이 컴퓨터의 폴더에만 적용됩니다.",
bottleneck_source_v2: "원본 읽기",
android_local_network_allow: "허용",
android_local_network_body: "허용하기 전까지 Android가 앱이 PS5에 연결하지 못하게 막습니다.",
+1 -1
View File
@@ -3589,7 +3589,7 @@ upload_as_image_title: "Albo wyślij jako jeden obraz gry",
"collection.add_server_folder": "Dodaj folder na {name}",
"collection.no_servers_hint": "Gry na NAS? Zapisz go w Połączeniach (SMB, FTP lub SFTP/WebDAV), a jego foldery będzie można dodać tutaj bez montowania.",
"collection.pick_server_folder": "Folder na {name} z twoimi grami",
"collection.server_folders_hint": "Folder na zapisanym serwerze (SMB, FTP, SFTP/WebDAV) jest czytany na miejscu: nic nie jest kopiowane na ten komputer, a jego gry instalują się prosto z serwera. Przenieś do Kosza i Uporządkuj działają tylko na folderach tego komputera.",
"collection.server_folders_hint": "Folder na zapisanym serwerze (SMB, FTP, SFTP/WebDAV) jest czytany na miejscu: nic nie jest kopiowane na ten komputer, a jego gry instalują się prosto z serwera. Uporządkuj działa także tam, zmieniając nazwy na samym serwerze; Przenieś do Kosza działa tylko na folderach tego komputera.",
bottleneck_source_v2: "odczyt źródła",
android_local_network_allow: "Zezwól",
android_local_network_body: "Android blokuje aplikacji dostęp do PS5, dopóki na to nie zezwolisz.",
+1 -1
View File
@@ -3351,7 +3351,7 @@ upload_as_image_title: "Ou envie como uma única imagem de jogo",
"collection.add_server_folder": "Adicionar uma pasta de {name}",
"collection.no_servers_hint": "Jogos em um NAS? Salve-o em Conexões (SMB, FTP ou SFTP/WebDAV) e as pastas dele podem ser adicionadas aqui sem montar.",
"collection.pick_server_folder": "A pasta de {name} que contém seus jogos",
"collection.server_folders_hint": "Uma pasta num servidor salvo (SMB, FTP, SFTP/WebDAV) é lida onde está: nada é copiado para este computador e os jogos dela instalam direto do servidor. Mover para a Lixeira e Organizar valem só para pastas deste computador.",
"collection.server_folders_hint": "Uma pasta num servidor salvo (SMB, FTP, SFTP/WebDAV) é lida onde está: nada é copiado para este computador e os jogos dela instalam direto do servidor. Organizar também funciona lá, renomeando no próprio servidor; Mover para a Lixeira vale só para pastas deste computador.",
bottleneck_source_v2: "a leitura da origem",
android_local_network_allow: "Permitir",
android_local_network_body: "O Android impede que o app alcance seu PS5 até você permitir.",
+1 -1
View File
@@ -3284,7 +3284,7 @@ upload_as_image_title: "Или отправить одним образом иг
"collection.add_server_folder": "Добавить папку на {name}",
"collection.no_servers_hint": "Игры на NAS? Сохраните его в «Подключениях» (SMB, FTP или SFTP/WebDAV), и его папки можно будет добавить сюда без монтирования.",
"collection.pick_server_folder": "Папка на {name} с вашими играми",
"collection.server_folders_hint": "Папка на сохранённом сервере (SMB, FTP, SFTP/WebDAV) читается на месте: на этот компьютер ничего не копируется, а её игры устанавливаются прямо с сервера. «В корзину» и «Упорядочить» работают только с папками этого компьютера.",
"collection.server_folders_hint": "Папка на сохранённом сервере (SMB, FTP, SFTP/WebDAV) читается на месте: на этот компьютер ничего не копируется, а её игры устанавливаются прямо с сервера. «Упорядочить» работает и там, переименовывая файлы на самом сервере; «В корзину» работает только с папками этого компьютера.",
bottleneck_source_v2: "чтение источника",
android_local_network_allow: "Разрешить",
android_local_network_body: "Android не даёт приложению связаться с PS5, пока вы это не разрешите.",
+1 -1
View File
@@ -3343,7 +3343,7 @@ upload_as_image_title: "หรือส่งเป็นอิมเมจเ
"collection.add_server_folder": "เพิ่มโฟลเดอร์บน {name}",
"collection.no_servers_hint": "มีเกมใน NAS? บันทึกไว้ในการเชื่อมต่อ (SMB, FTP หรือ SFTP/WebDAV) แล้วเพิ่มโฟลเดอร์ของมันที่นี่ได้โดยไม่ต้องเมานต์",
"collection.pick_server_folder": "โฟลเดอร์บน {name} ที่เก็บเกมของคุณ",
"collection.server_folders_hint": "โฟลเดอร์บนเซิร์ฟเวอร์ที่บันทึกไว้ (SMB, FTP, SFTP/WebDAV) อ่านจากที่เดิม: ไม่มีการคัดลอกมายังคอมพิวเตอร์นี้ และเกมติดตั้งจากเซิร์ฟเวอร์โดยตรง ย้ายไปถังขยะและการจัดใช้ได้กับโฟลเดอร์บนคอมพิวเตอร์นี้เท่านั้น",
"collection.server_folders_hint": "โฟลเดอร์บนเซิร์ฟเวอร์ที่บันทึกไว้ (SMB, FTP, SFTP/WebDAV) อ่านจากที่เดิม: ไม่มีการคัดลอกมายังคอมพิวเตอร์นี้ และเกมติดตั้งจากเซิร์ฟเวอร์โดยตรง การจัดใช้ได้ที่นั่นด้วย โดยเปลี่ยนชื่อบนเซิร์ฟเวอร์เอง ส่วนย้ายไปถังขยะใช้ได้กับโฟลเดอร์บนคอมพิวเตอร์นี้เท่านั้น",
bottleneck_source_v2: "การอ่านต้นทาง",
android_local_network_allow: "อนุญาต",
android_local_network_body: "Android จะบล็อกไม่ให้แอปเชื่อมต่อ PS5 จนกว่าคุณจะอนุญาต",
+1 -1
View File
@@ -3341,7 +3341,7 @@ upload_as_image_title: "Ya da tek bir oyun imajı olarak gönder",
"collection.add_server_folder": "{name} üzerinde klasör ekle",
"collection.no_servers_hint": "Oyunlar bir NAS'ta mı? Bağlantılar'a kaydet (SMB, FTP veya SFTP/WebDAV); klasörleri bağlamadan buraya eklenebilir.",
"collection.pick_server_folder": "{name} üzerindeki oyunlarının bulunduğu klasör",
"collection.server_folders_hint": "Kayıtlı bir sunucudaki (SMB, FTP, SFTP/WebDAV) klasör yerinde okunur: bu bilgisayara hiçbir şey kopyalanmaz ve oyunları doğrudan sunucudan yüklenir. Çöp Sepeti'ne taşı ve Düzenle yalnızca bu bilgisayarın klasörlerinde çalışır.",
"collection.server_folders_hint": "Kayıtlı bir sunucudaki (SMB, FTP, SFTP/WebDAV) klasör yerinde okunur: bu bilgisayara hiçbir şey kopyalanmaz ve oyunları doğrudan sunucudan yüklenir. Düzenle orada da çalışır ve adları doğrudan sunucuda değiştirir; Çöp Sepeti'ne taşı yalnızca bu bilgisayarın klasörlerinde çalışır.",
bottleneck_source_v2: "kaynağın okunması",
android_local_network_allow: "İzin ver",
android_local_network_body: "İzin verene kadar Android, uygulamanın PS5'ine ulaşmasını engeller.",
+1 -1
View File
@@ -3336,7 +3336,7 @@ upload_as_image_title: "Hoặc gửi dưới dạng một tệp ảnh trò chơi
"collection.add_server_folder": "Thêm thư mục trên {name}",
"collection.no_servers_hint": "Trò chơi trên NAS? Lưu nó trong Kết nối (SMB, FTP hoặc SFTP/WebDAV), rồi có thể thêm thư mục của nó vào đây mà không cần gắn.",
"collection.pick_server_folder": "Thư mục trên {name} chứa trò chơi của bạn",
"collection.server_folders_hint": "Thư mục trên máy chủ đã lưu (SMB, FTP, SFTP/WebDAV) được đọc tại chỗ: không có gì được sao chép về máy tính này, và trò chơi được cài thẳng từ máy chủ. Chuyển vào Thùng rác và Sắp xếp chỉ áp dụng cho thư mục trên máy tính này.",
"collection.server_folders_hint": "Thư mục trên máy chủ đã lưu (SMB, FTP, SFTP/WebDAV) được đọc tại chỗ: không có gì được sao chép về máy tính này, và trò chơi được cài thẳng từ máy chủ. Sắp xếp cũng dùng được ở đó, đổi tên ngay trên máy chủ; Chuyển vào Thùng rác chỉ áp dụng cho thư mục trên máy tính này.",
bottleneck_source_v2: "đọc nguồn",
android_local_network_allow: "Cho phép",
android_local_network_body: "Android chặn ứng dụng kết nối tới PS5 cho tới khi bạn cho phép.",
+1 -1
View File
@@ -3290,7 +3290,7 @@ upload_as_image_title: "或作为一个游戏镜像发送",
"collection.add_server_folder": "添加 {name} 上的文件夹",
"collection.no_servers_hint": "游戏在 NAS 上?在“连接”中保存它(SMB、FTP 或 SFTP/WebDAV),就能不挂载直接在这里添加它的文件夹。",
"collection.pick_server_folder": "{name} 上存放游戏的文件夹",
"collection.server_folders_hint": "已保存服务器(SMB、FTP、SFTP/WebDAV)上的文件夹直接就地读取:不会复制到这台电脑,其中的游戏直接从服务器安装。“移到废纸篓”和“整理”只适用于这台电脑上的文件夹。",
"collection.server_folders_hint": "已保存服务器(SMB、FTP、SFTP/WebDAV)上的文件夹直接就地读取:不会复制到这台电脑,其中的游戏直接从服务器安装。“整理”在服务器上同样可用,直接在服务器上重命名;“移到废纸篓”只适用于这台电脑上的文件夹。",
bottleneck_source_v2: "读取源文件",
android_local_network_allow: "允许",
android_local_network_body: "在你允许之前,Android 会阻止此应用连接你的 PS5。",
+1 -1
View File
@@ -3284,7 +3284,7 @@ upload_as_image_title: "或作為一個遊戲映像檔傳送",
"collection.add_server_folder": "新增 {name} 上的資料夾",
"collection.no_servers_hint": "遊戲在 NAS 上?在「連線」中儲存它(SMB、FTP 或 SFTP/WebDAV),就能不掛載直接在這裡新增它的資料夾。",
"collection.pick_server_folder": "{name} 上存放遊戲的資料夾",
"collection.server_folders_hint": "已儲存伺服器(SMB、FTP、SFTP/WebDAV)上的資料夾直接就地讀取:不會複製到這台電腦,其中的遊戲直接從伺服器安裝。「移到垃圾桶」和「整理」只適用於這台電腦上的資料夾。",
"collection.server_folders_hint": "已儲存伺服器(SMB、FTP、SFTP/WebDAV)上的資料夾直接就地讀取:不會複製到這台電腦,其中的遊戲直接從伺服器安裝。「整理」在伺服器上同樣可用,直接在伺服器上重新命名;「移到垃圾桶」只適用於這台電腦上的資料夾。",
bottleneck_source_v2: "讀取來源",
android_local_network_allow: "允許",
android_local_network_body: "在你允許之前,Android 會阻止此 App 連線到你的 PS5。",
@@ -158,7 +158,7 @@ export function FoldersModal({
? tr(
"collection.server_folders_hint",
undefined,
"A folder on a saved server (SMB, FTP, SFTP) is read in place: nothing is copied to this computer, and its games install straight from the server. Move to Trash and Organize work on this computer's folders only.",
"A folder on a saved server (SMB, FTP, SFTP/WebDAV) is read in place: nothing is copied to this computer, and its games install straight from the server. Organize works there too, renaming on the server itself; Move to Trash works on this computer's folders only.",
)
: tr(
"collection.no_servers_hint",
@@ -66,6 +66,9 @@ export function OrganizeModal({
(s) => s.reason !== "already correct",
);
const inPlace = (plan?.skipped.length ?? 0) - notInPlace.length;
/** A container as shown: "Server: /path" for a folder on a saved server. */
const where = (container: string) => plan?.labels?.[container] ?? container;
const manyPlaces = (plan?.containers.length ?? 0) > 1;
const apply = async () => {
if (!plan || chosen.length === 0) return;
@@ -115,6 +118,7 @@ export function OrganizeModal({
"Undone: {n} restored, {problems} left where they are",
),
);
if (r.note) pushNotification("info", r.note);
await loadCollection();
await refresh();
} catch (e) {
@@ -174,9 +178,10 @@ export function OrganizeModal({
)}
{result.failed.map((f) => (
<div
key={f.from}
key={`${f.container}|${f.from}`}
className="mt-1 break-all text-[var(--color-bad)]"
>
{manyPlaces ? `${where(f.container)} › ` : ""}
{f.from}: {f.error}
</div>
))}
@@ -241,11 +246,17 @@ export function OrganizeModal({
</button>
</div>
<ul className="mt-2 flex max-h-[45vh] flex-col gap-1.5 overflow-y-auto pr-1">
{plan.moves.map((m) => (
{plan.moves.map((m, i) => (
<li
key={moveKey(m)}
className="rounded-2xl border border-[var(--glass-edge)] bg-[var(--color-surface)] p-2 text-xs"
>
{(i === 0 ||
plan.moves[i - 1].container !== m.container) && (
<div className="mb-1 truncate text-[0.6875rem] font-medium text-[var(--color-accent)]">
{where(m.container)}
</div>
)}
<div className="flex items-start gap-2">
<Checkbox
checked={!unticked.has(moveKey(m))}
@@ -325,6 +336,9 @@ export function OrganizeModal({
<li key={`${s.container}|${s.path}`} className="break-all">
<span className="text-[var(--color-warn)]">{s.reason}</span>
{": "}
{manyPlaces && s.container && (
<span>{where(s.container)} › </span>
)}
<span className="font-mono">{s.path}</span>
{s.detail ? ` (${s.detail})` : ""}
</li>
@@ -378,8 +392,11 @@ export function OrganizeModal({
<span className="text-[var(--color-muted)]">
{tr("collection.org_run_moved", { n: r.moved }, "{n} moved")}
</span>
<span className="min-w-0 flex-1 truncate font-mono text-[0.6875rem] text-[var(--color-muted)]">
{r.container_root}
<span
className="min-w-0 flex-1 truncate font-mono text-[0.6875rem] text-[var(--color-muted)]"
title={r.container_root}
>
{r.label ?? r.container_root}
</span>
{r.reverted_at ? (
<span className="text-[var(--color-muted)]">
@@ -19,6 +19,7 @@ pub mod identify;
pub mod junk;
pub mod online;
pub mod organize;
pub mod organize_fs;
pub mod scan;
pub mod store;
@@ -7,6 +7,11 @@
//! the preview and the run come from the same code, and the run re-plans and drops any move
//! that changed since the preview. Nothing is overwritten, nothing is deleted, and every move
//! is a rename on one volume (never a copy). Each run writes its undo log before its first move.
//!
//! The folder can be on this computer or on a saved server: everything here goes through a
//! [`Tree`], and a server's paths are its `remote://` paths. On a server a rename never moves
//! between servers (a move stays inside its container), and a dropped connection stops the run
//! where it is, with the log saying what happened so far.
use std::collections::{BTreeMap, HashMap, HashSet};
use std::path::{Path, PathBuf};
@@ -15,6 +20,7 @@ use serde::{Deserialize, Serialize};
use unicode_normalization::UnicodeNormalization;
use super::junk;
use super::organize_fs::{path_string, OpError, Tree};
use super::PkgInfo;
pub const DUPLICATE_TAG: &str = " - DUPLICATE";
@@ -92,9 +98,18 @@ pub struct Plan {
/// name decomposed while a package's param.json composes it; without this a title with a
/// macron would look renamed on every run.
fn key(p: &Path) -> String {
p.to_string_lossy().nfc().collect::<String>().to_lowercase()
p.components()
.map(|c| c.as_os_str().to_string_lossy().into_owned())
.collect::<Vec<_>>()
.join("/")
.nfc()
.collect::<String>()
.to_lowercase()
}
/// Deeper than any library arrangement; a bound for a walk over a server.
const MAX_DEPTH: usize = 12;
fn rel(container: &Path, p: &Path) -> String {
p.strip_prefix(container)
.unwrap_or(p)
@@ -279,7 +294,7 @@ fn split_part(name: &str) -> Option<(&str, u32)> {
}
/// Every part of a split set (`name_0.pkg`, `name_1.pkg`, …) in order, or just `[path]`.
pub fn split_siblings(path: &Path) -> Vec<PathBuf> {
pub fn split_siblings(tree: &dyn Tree, path: &Path) -> Vec<PathBuf> {
let Some(name) = path.file_name().and_then(|n| n.to_str()) else {
return vec![path.to_path_buf()];
};
@@ -287,14 +302,14 @@ pub fn split_siblings(path: &Path) -> Vec<PathBuf> {
return vec![path.to_path_buf()];
};
let dir = path.parent().unwrap_or(Path::new("."));
let mut sibs: Vec<(u32, PathBuf)> = std::fs::read_dir(dir)
let mut sibs: Vec<(u32, PathBuf)> = tree
.list(dir)
.unwrap_or_default()
.into_iter()
.flatten()
.flatten()
.filter(|e| e.is_file)
.filter_map(|e| {
let n = e.file_name().to_string_lossy().into_owned();
let (s, k) = split_part(&n)?;
(s == stem && !junk::is_junk_name(&n)).then(|| (k, e.path()))
let (s, k) = split_part(&e.name)?;
(s == stem && !junk::is_junk_name(&e.name)).then(|| (k, dir.join(&e.name)))
})
.collect();
sibs.sort_by_key(|(k, _)| *k);
@@ -306,27 +321,24 @@ pub fn split_siblings(path: &Path) -> Vec<PathBuf> {
}
/// Every package below a folder, Finder's sidecars and dot-folders ignored.
pub fn scan(container: &Path) -> Vec<PathBuf> {
pub fn scan(tree: &dyn Tree, container: &Path) -> Vec<PathBuf> {
let mut found = Vec::new();
let mut stack = vec![container.to_path_buf()];
while let Some(dir) = stack.pop() {
let Ok(rd) = std::fs::read_dir(&dir) else {
let mut stack = vec![(container.to_path_buf(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
let Some(mut entries) = tree.list(&dir) else {
continue;
};
let mut entries: Vec<_> = rd.flatten().collect();
entries.sort_by_key(|e| e.file_name());
entries.sort_by(|a, b| a.name.cmp(&b.name));
for e in entries {
let name = e.file_name().to_string_lossy().into_owned();
let Ok(ft) = e.file_type() else { continue };
if ft.is_dir() {
if !name.starts_with('.') {
stack.push(e.path());
if e.is_dir {
if !e.name.starts_with('.') && depth < MAX_DEPTH {
stack.push((dir.join(&e.name), depth + 1));
}
} else if ft.is_file()
&& !junk::is_junk_name(&name)
&& name.to_ascii_lowercase().ends_with(".pkg")
} else if e.is_file
&& !junk::is_junk_name(&e.name)
&& e.name.to_ascii_lowercase().ends_with(".pkg")
{
found.push(e.path());
found.push(dir.join(&e.name));
}
}
}
@@ -337,20 +349,19 @@ pub fn scan(container: &Path) -> Vec<PathBuf> {
/// The packages lying loose in a folder itself, not in its subfolders. A collection root is
/// organized this way: its subfolders are containers of their own (planned separately), so a
/// root plans only the packages at its top level.
pub fn scan_top(dir: &Path) -> Vec<PathBuf> {
let Ok(rd) = std::fs::read_dir(dir) else {
pub fn scan_top(tree: &dyn Tree, dir: &Path) -> Vec<PathBuf> {
let Some(entries) = tree.list(dir) else {
return Vec::new();
};
let mut found: Vec<PathBuf> = rd
.flatten()
let mut found: Vec<PathBuf> = entries
.into_iter()
.filter(|e| {
let name = e.file_name().to_string_lossy().into_owned();
e.file_type().is_ok_and(|t| t.is_file())
&& !name.starts_with('.')
&& !junk::is_junk_name(&name)
&& name.to_ascii_lowercase().ends_with(".pkg")
e.is_file
&& !e.name.starts_with('.')
&& !junk::is_junk_name(&e.name)
&& e.name.to_ascii_lowercase().ends_with(".pkg")
})
.map(|e| e.path())
.map(|e| dir.join(&e.name))
.collect();
found.sort();
found
@@ -358,20 +369,17 @@ pub fn scan_top(dir: &Path) -> Vec<PathBuf> {
/// The top-level folders of a collection root that hold packages, whatever they are called,
/// and the root itself when packages lie loose in it.
pub fn containers(root: &Path) -> Vec<PathBuf> {
let Ok(rd) = std::fs::read_dir(root) else {
pub fn containers(tree: &dyn Tree, root: &Path) -> Vec<PathBuf> {
let Some(entries) = tree.list(root) else {
return Vec::new();
};
let mut out: Vec<PathBuf> = rd
.flatten()
.filter(|e| {
!e.file_name().to_string_lossy().starts_with('.')
&& e.file_type().is_ok_and(|t| t.is_dir())
})
.map(|e| e.path())
.filter(|p| !scan(p).is_empty())
let mut out: Vec<PathBuf> = entries
.into_iter()
.filter(|e| !e.name.starts_with('.') && e.is_dir)
.map(|e| root.join(&e.name))
.filter(|p| !scan(tree, p).is_empty())
.collect();
if !scan_top(root).is_empty() {
if !scan_top(tree, root).is_empty() {
out.push(root.to_path_buf());
}
out.sort();
@@ -380,19 +388,18 @@ pub fn containers(root: &Path) -> Vec<PathBuf> {
/// Reads a package's facts, taking them from `cached(path, size, mtime)` when it knows the
/// file as it is now (the Collection's index), and from the file otherwise.
pub fn read_facts_with(path: &Path, cached: &dyn Fn(&Path, u64, i64) -> Option<PkgInfo>) -> Facts {
let parts = split_siblings(path);
/// On a server the index's facts save reading each package's header over the network.
pub fn read_facts_with(
tree: &dyn Tree,
path: &Path,
cached: &dyn Fn(&Path, u64, i64) -> Option<PkgInfo>,
) -> Facts {
let parts = split_siblings(tree, path);
let size = parts
.iter()
.map(|p| std::fs::metadata(p).map(|m| m.len()).unwrap_or(0))
.map(|p| tree.meta(p).map(|m| m.size).unwrap_or(0))
.sum();
let first = std::fs::metadata(&parts[0]).ok();
let mtime = first
.as_ref()
.and_then(|m| m.modified().ok())
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
.unwrap_or(0);
let mtime = tree.meta(&parts[0]).map(|m| m.mtime).unwrap_or(0);
let info = (parts.len() == 1)
.then(|| cached(&parts[0], size, mtime))
.flatten()
@@ -407,19 +414,20 @@ pub fn read_facts_with(path: &Path, cached: &dyn Fn(&Path, u64, i64) -> Option<P
}
/// Every file name in the folder that a move must not land on: everything that stays.
fn taken_names(container: &Path, moving: &HashSet<String>) -> HashSet<String> {
fn taken_names(tree: &dyn Tree, container: &Path, moving: &HashSet<String>) -> HashSet<String> {
let mut taken = HashSet::new();
let mut stack = vec![container.to_path_buf()];
while let Some(dir) = stack.pop() {
for e in std::fs::read_dir(&dir).into_iter().flatten().flatten() {
let name = e.file_name().to_string_lossy().into_owned();
let Ok(ft) = e.file_type() else { continue };
if ft.is_dir() {
if !name.starts_with('.') {
stack.push(e.path());
let mut stack = vec![(container.to_path_buf(), 0usize)];
while let Some((dir, depth)) = stack.pop() {
for e in tree.list(&dir).unwrap_or_default() {
let path = dir.join(&e.name);
if e.is_dir {
// A folder's name is taken too: no file may land on it.
taken.insert(key(&path));
if !e.name.starts_with('.') && depth < MAX_DEPTH {
stack.push((path, depth + 1));
}
} else if !junk::is_junk_name(&name) {
let k = key(&e.path());
} else if !junk::is_junk_name(&e.name) {
let k = key(&path);
if !moving.contains(&k) {
taken.insert(k);
}
@@ -487,18 +495,23 @@ fn unique_names(
/// Pure: what organizing `container` would do. `facts` reads one package; nothing is touched.
#[cfg(test)]
pub fn plan_with(container: &Path, facts: &dyn Fn(&Path) -> Facts) -> Plan {
plan_files(container, scan(container), facts)
pub fn plan_with(tree: &dyn Tree, container: &Path, facts: &dyn Fn(&Path) -> Facts) -> Plan {
plan_files(tree, container, scan(tree, container), facts)
}
/// `plan_with` over the given packages of `container` (all of them, or a root's loose ones).
pub fn plan_files(container: &Path, files: Vec<PathBuf>, facts: &dyn Fn(&Path) -> Facts) -> Plan {
let cname = container.to_string_lossy().into_owned();
pub fn plan_files(
tree: &dyn Tree,
container: &Path,
files: Vec<PathBuf>,
facts: &dyn Fn(&Path) -> Facts,
) -> Plan {
let cname = path_string(container);
let mut out = Plan::default();
let mut parsed: Vec<Facts> = Vec::new();
for path in files {
// Only part 0 of a split set carries a header: plan the set from it.
let parts = split_siblings(&path);
let parts = split_siblings(tree, &path);
if parts.len() > 1 && parts[0] != path {
continue;
}
@@ -530,7 +543,7 @@ pub fn plan_files(container: &Path, files: Vec<PathBuf>, facts: &dyn Fn(&Path) -
.iter()
.flat_map(|f| f.parts.iter().map(|p| key(p)))
.collect();
let mut taken = taken_names(container, &moving);
let mut taken = taken_names(tree, container, &moving);
for (title_id, members) in groups {
let title = group_title(&members);
@@ -649,7 +662,7 @@ pub fn plan_files(container: &Path, files: Vec<PathBuf>, facts: &dyn Fn(&Path) -
reason: f.info.kind_reason.clone(),
version: f.info.version.clone(),
region: f.info.region.clone(),
size: std::fs::metadata(part).map(|m| m.len()).unwrap_or(0),
size: tree.meta(part).map(|m| m.size).unwrap_or(0),
content_id: f.info.content_id.clone(),
part: (names.len() > 1).then(|| format!("{} of {}", i + 1, names.len())),
duplicate: duplicate_of.is_some(),
@@ -665,35 +678,60 @@ pub fn plan_files(container: &Path, files: Vec<PathBuf>, facts: &dyn Fn(&Path) -
out
}
/// A plan whose package facts come from the Collection's index where it is current.
pub fn plan_cached(container: &Path, lib: Option<&super::Library>, roots: &[PathBuf]) -> Plan {
/// A plan whose package facts come from the Collection's index where it is current. `roots`
/// are (the root as a tree path, the root as the index keys it).
pub fn plan_cached(
tree: &dyn Tree,
container: &Path,
lib: Option<&super::Library>,
roots: &[(PathBuf, String)],
) -> Plan {
let cached = |p: &Path, size: u64, mtime: i64| -> Option<PkgInfo> {
let lib = lib?;
roots.iter().find_map(|root| {
roots.iter().find_map(|(root, key_root)| {
let rel = p.strip_prefix(root).ok()?;
let rel = rel
.components()
.map(|c| c.as_os_str().to_string_lossy().into_owned())
.collect::<Vec<_>>()
.join("/");
let c = lib.pkg_cache.get(&format!("{}|{rel}", root.display()))?;
let c = lib.pkg_cache.get(&format!("{key_root}|{rel}"))?;
(c.size == size && c.mtime == mtime && c.v == super::IDENTIFY_VERSION)
.then(|| c.pkg.clone())
})
};
// A root is planned for its loose packages only: its subfolders are containers of their own.
let files = if roots.iter().any(|r| r == container) {
scan_top(container)
let files = if roots.iter().any(|(r, _)| r == container) {
scan_top(tree, container)
} else {
scan(container)
scan(tree, container)
};
plan_files(container, files, &|p: &Path| read_facts_with(p, &cached))
let mut plan = plan_files(tree, container, files, &|p: &Path| {
read_facts_with(tree, p, &cached)
});
if let Some(trouble) = tree.trouble() {
// A folder the server would not list: its packages are missing from this preview.
plan.skipped.push(Skipped {
container: path_string(container),
path: String::new(),
reason: SKIP_UNREADABLE.into(),
detail: trouble,
});
}
plan
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UndoLog {
pub version: u32,
pub container_root: String,
/// The saved server's name when the folder is on one, for the list of runs.
#[serde(default, skip_serializing_if = "String::is_empty")]
pub server: String,
/// Which server the folder was on. Undo refuses when the saved connection (same id) now
/// points somewhere else. Logs from before it was recorded have none.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub origin: Option<Origin>,
pub created_at: String,
pub moves: Vec<Move>,
#[serde(default)]
@@ -709,11 +747,48 @@ pub struct UndoLog {
pub reverted_at: Option<String>,
}
/// A server folder's whereabouts, as the saved connection said when the run was made.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct Origin {
pub protocol: String,
/// Lowercase.
pub host: String,
pub port: u16,
/// The SMB share, lowercase ("" for other protocols).
#[serde(default)]
pub share: String,
/// The folder's server path.
pub path: String,
}
impl Origin {
/// The same server: protocol, host, port and share.
pub fn same_server(&self, other: &Origin) -> bool {
self.protocol == other.protocol
&& self.host == other.host
&& self.port == other.port
&& self.share == other.share
}
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct FailedMove {
#[serde(flatten)]
pub mv: Move,
pub error: String,
/// The connection dropped during this move: it may have happened. Undo looks.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub uncertain: bool,
}
impl FailedMove {
fn new(mv: Move, error: impl Into<String>) -> Self {
Self {
mv,
error: error.into(),
uncertain: false,
}
}
}
fn now_iso() -> String {
@@ -758,19 +833,19 @@ fn write_log(path: &Path, log: &UndoLog) -> std::io::Result<()> {
/// Removes a folder holding nothing but Finder's files. True when it is gone afterwards.
/// Finder can rewrite `.DS_Store` the moment it notices a folder, so it retries briefly.
fn try_remove_dir(dir: &Path) -> bool {
fn try_remove_dir(tree: &dyn Tree, dir: &Path) -> bool {
for _ in 0..3 {
let Ok(rd) = std::fs::read_dir(dir) else {
return !dir.exists();
let Some(entries) = tree.list(dir) else {
return tree.exists(dir) == Ok(false);
};
let entries: Vec<PathBuf> = rd.flatten().map(|e| e.path()).collect();
if entries.iter().any(|p| !junk::is_removable_junk(p)) {
let entries: Vec<PathBuf> = entries.iter().map(|e| dir.join(&e.name)).collect();
if entries.iter().any(|p| !tree.is_removable_junk(p)) {
return false;
}
for p in &entries {
let _ = std::fs::remove_file(p);
tree.remove_junk(p);
}
if std::fs::remove_dir(dir).is_ok() || !dir.exists() {
if tree.remove_dir(dir).is_ok() || tree.exists(dir) == Ok(false) {
return true;
}
}
@@ -779,20 +854,27 @@ fn try_remove_dir(dir: &Path) -> bool {
/// Source folders the moves emptied, removed up to the container. Returns (removed, kept):
/// kept names one that held only Finder's files and still could not be removed.
fn prune_empty(container: &Path, started_in: &HashSet<PathBuf>) -> (Vec<String>, Vec<String>) {
fn prune_empty(
tree: &dyn Tree,
container: &Path,
started_in: &HashSet<PathBuf>,
) -> (Vec<String>, Vec<String>) {
let (mut removed, mut kept) = (Vec::new(), Vec::new());
let mut dirs: Vec<&PathBuf> = started_in.iter().collect();
dirs.sort_by_key(|d| std::cmp::Reverse(d.as_os_str().len()));
for start in dirs {
let mut dir = start.clone();
while dir != container && dir.starts_with(container) {
let Ok(rd) = std::fs::read_dir(&dir) else {
let Some(entries) = tree.list(&dir) else {
break;
};
if rd.flatten().any(|e| !junk::is_removable_junk(&e.path())) {
if entries
.iter()
.any(|e| !tree.is_removable_junk(&dir.join(&e.name)))
{
break;
}
if try_remove_dir(&dir) {
if try_remove_dir(tree, &dir) {
removed.push(rel(container, &dir));
} else {
kept.push(rel(container, &dir));
@@ -807,14 +889,48 @@ fn prune_empty(container: &Path, started_in: &HashSet<PathBuf>) -> (Vec<String>,
(removed, kept)
}
/// Makes `dir` and the folders above it, down from `container` (which exists). Folders known
/// to exist are skipped, so a run of moves into one folder makes it once.
fn make_dirs(
tree: &dyn Tree,
container: &Path,
dir: &Path,
known: &mut HashSet<PathBuf>,
) -> Result<(), OpError> {
let Ok(below) = dir.strip_prefix(container) else {
return Err(OpError::Other("path leaves the folder".into()));
};
let mut at = container.to_path_buf();
for c in below.components() {
at.push(c);
if known.contains(&at) {
continue;
}
tree.create_dir(&at)?;
known.insert(at.clone());
}
Ok(())
}
/// How many moves run between rewrites of the log.
const LOG_EVERY: usize = 16;
/// Runs `moves` (all in `container`), the log written before the first one.
pub fn apply(container: &Path, moves: Vec<Move>, log_path: &Path) -> std::io::Result<UndoLog> {
/// Runs `moves` (all in `container`), the log written before the first one. A dropped
/// connection (one a fresh sign-in did not cure) stops the run: the move in flight is logged as
/// uncertain, the rest as not tried, and the log stays good for Undo.
pub fn apply(
tree: &dyn Tree,
container: &Path,
moves: Vec<Move>,
log_path: &Path,
server: &str,
origin: Option<Origin>,
) -> std::io::Result<UndoLog> {
let mut log = UndoLog {
version: 1,
container_root: container.to_string_lossy().into_owned(),
container_root: path_string(container),
server: server.to_string(),
origin,
created_at: now_iso(),
moves: moves.clone(),
completed: Vec::new(),
@@ -825,40 +941,54 @@ pub fn apply(container: &Path, moves: Vec<Move>, log_path: &Path) -> std::io::Re
};
write_log(log_path, &log)?;
let mut started_in = HashSet::new();
let mut made = HashSet::new();
let mut lost: Option<String> = None;
for mv in moves {
if !contained(&mv.from) || !contained(&mv.to) {
log.failed.push(FailedMove {
if let Some(why) = &lost {
log.failed.push(FailedMove::new(
mv,
error: "path leaves the folder".into(),
});
format!("not tried: the server stopped answering ({why})"),
));
continue;
}
if !contained(&mv.from) || !contained(&mv.to) {
log.failed
.push(FailedMove::new(mv, "path leaves the folder"));
continue;
}
let src = container.join(&mv.from);
let dst = container.join(&mv.to);
if dst.exists() {
log.failed.push(FailedMove {
mv,
error: "destination appeared".into(),
});
continue;
}
let res = dst
.parent()
.map_or(Ok(()), std::fs::create_dir_all)
// A rename, never a copy: a move across volumes fails instead of copying.
.and_then(|()| std::fs::rename(&src, &dst));
if let Err(e) = res {
log.failed.push(FailedMove {
mv,
error: e.to_string(),
});
continue;
let res = match tree.exists(&dst) {
Ok(true) => Err(OpError::Exists("destination appeared".into())),
Ok(false) => dst
.parent()
.map_or(Ok(()), |d| make_dirs(tree, container, d, &mut made))
.and_then(|()| tree.rename_new(&src, &dst)),
Err(e) => Err(e),
};
match res {
Ok(()) => {}
Err(OpError::Lost(e)) => {
lost = Some(e.clone());
log.failed.push(FailedMove {
mv,
error: format!("the server stopped answering during this move ({e})"),
uncertain: true,
});
let _ = write_log(log_path, &log);
continue;
}
Err(OpError::Exists(_)) => {
log.failed.push(FailedMove::new(mv, "destination appeared"));
continue;
}
Err(OpError::Other(e)) => {
log.failed.push(FailedMove::new(mv, e));
continue;
}
}
if let (Some(dir), Some(name)) = (src.parent(), src.file_name()) {
let sidecar = dir.join(format!("._{}", name.to_string_lossy()));
if junk::is_sidecar(&sidecar) {
let _ = std::fs::remove_file(sidecar);
}
tree.remove_junk(&dir.join(format!("._{}", name.to_string_lossy())));
started_in.insert(dir.to_path_buf());
}
log.completed.push(mv);
@@ -867,21 +997,30 @@ pub fn apply(container: &Path, moves: Vec<Move>, log_path: &Path) -> std::io::Re
let _ = write_log(log_path, &log);
}
}
let (removed, kept) = prune_empty(container, &started_in);
log.removed_dirs = removed;
log.kept_dirs = kept;
if lost.is_none() {
let (removed, kept) = prune_empty(tree, container, &started_in);
log.removed_dirs = removed;
log.kept_dirs = kept;
}
write_log(log_path, &log)?;
Ok(log)
}
/// Gives the tree a path is in.
pub type OpenTree = dyn Fn(&Path) -> Result<Box<dyn Tree>, String>;
#[derive(Debug, Clone, Default, Serialize)]
pub struct RevertResult {
pub reverted: Vec<Move>,
pub problems: Vec<FailedMove>,
/// Something the user should know about how it was undone.
#[serde(skip_serializing_if = "Option::is_none")]
pub note: Option<String>,
}
/// Undoes a run from its log: only the moves that happened, newest first.
pub fn revert(log_path: &Path) -> std::io::Result<RevertResult> {
/// Undoes a run from its log: only the moves that happened, newest first. `open` gives the
/// tree the run's folder is in (this computer, or its server).
pub fn revert(log_path: &Path, open: &OpenTree) -> std::io::Result<RevertResult> {
let bytes = std::fs::read(log_path)?;
let mut log: UndoLog = serde_json::from_slice(&bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?;
@@ -889,53 +1028,88 @@ pub fn revert(log_path: &Path) -> std::io::Result<RevertResult> {
return Err(std::io::Error::other("this run was already undone"));
}
let root = PathBuf::from(&log.container_root);
let tree = open(&root).map_err(std::io::Error::other)?;
let tree = tree.as_ref();
// The server must answer before anything is judged "gone".
if !tree
.exists(&root)
.map_err(|e| std::io::Error::other(e.to_string()))?
{
return Err(std::io::Error::other(format!(
"{} is not there any more",
log.container_root
)));
}
let mut out = RevertResult::default();
// A run that died after its last log write left moves that happened but are not in
// `completed`; those whose file sits at the new name and not the old one are undone too.
let here = |p: &Path| tree.exists(p);
// A run that died after its last log write, or lost its server mid-move, left moves that
// may have happened but are not in `completed`; those whose file sits at the new name and
// not the old one are undone too.
let mut done = log.completed.clone();
for mv in &log.moves {
if done.contains(mv) || log.failed.iter().any(|f| &f.mv == mv) {
if done.contains(mv) || log.failed.iter().any(|f| &f.mv == mv && !f.uncertain) {
continue;
}
if contained(&mv.from)
&& contained(&mv.to)
&& root.join(&mv.to).exists()
&& !root.join(&mv.from).exists()
&& here(&root.join(&mv.to)) == Ok(true)
&& here(&root.join(&mv.from)) == Ok(false)
{
done.push(mv.clone());
}
}
let mut made = HashSet::new();
let mut lost = None;
for mv in done.iter().rev() {
let src = root.join(&mv.to);
let dst = root.join(&mv.from);
let problem = if !contained(&mv.from) || !contained(&mv.to) {
let problem = if let Some(why) = &lost {
Some(format!("not tried: the server stopped answering ({why})"))
} else if !contained(&mv.from) || !contained(&mv.to) {
Some("path leaves the folder".to_string())
} else if !src.exists() {
Some("moved file is gone".to_string())
} else if dst.exists() {
Some("original name is taken".to_string())
} else {
dst.parent()
.map_or(Ok(()), std::fs::create_dir_all)
.and_then(|()| std::fs::rename(&src, &dst))
.err()
.map(|e| e.to_string())
match (here(&src), here(&dst)) {
(Ok(false), _) => Some("moved file is gone".to_string()),
(_, Ok(true)) => Some("original name is taken".to_string()),
(Ok(true), Ok(false)) => dst
.parent()
.map_or(Ok(()), |d| make_dirs(tree, &root, d, &mut made))
.and_then(|()| tree.rename_new(&src, &dst))
.err()
.map(|e| {
if let OpError::Lost(m) = &e {
lost = Some(m.clone());
}
e.to_string()
}),
(Err(e), _) | (_, Err(e)) => {
if let OpError::Lost(m) = &e {
lost = Some(m.clone());
}
Some(e.to_string())
}
}
};
match problem {
Some(error) => out.problems.push(FailedMove {
mv: mv.clone(),
error,
}),
Some(error) => out.problems.push(FailedMove::new(mv.clone(), error)),
None => out.reverted.push(mv.clone()),
}
}
if let Some(why) = lost {
// Left undoable: the moves already put back now sit at their old names, so the next
// Undo skips them ("moved file is gone") and finishes the rest.
return Err(std::io::Error::other(format!(
"the server stopped answering part way ({} put back); undo again once it is back: {why}",
out.reverted.len()
)));
}
// Folders the run created and the undo emptied.
let made: HashSet<PathBuf> = out
.reverted
.iter()
.filter_map(|m| root.join(&m.to).parent().map(Path::to_path_buf))
.collect();
prune_empty(&root, &made);
prune_empty(tree, &root, &made);
log.reverted_at = Some(now_iso());
write_log(log_path, &log)?;
Ok(out)
@@ -943,8 +1117,13 @@ pub fn revert(log_path: &Path) -> std::io::Result<RevertResult> {
#[cfg(test)]
mod tests {
use super::super::organize_fs::LocalTree;
use super::*;
fn local(_: &Path) -> Result<Box<dyn Tree>, String> {
Ok(Box::new(LocalTree))
}
struct Lib(PathBuf);
impl Drop for Lib {
fn drop(&mut self) {
@@ -964,7 +1143,7 @@ mod tests {
fn table<'a>(entries: &'a [(&'a str, PkgInfo)]) -> impl Fn(&Path) -> Facts + 'a {
move |p: &Path| {
let name = p.file_name().unwrap().to_string_lossy().into_owned();
let parts = split_siblings(p);
let parts = split_siblings(&LocalTree, p);
let size = parts
.iter()
.map(|q| std::fs::metadata(q).map(|m| m.len()).unwrap_or(0))
@@ -1031,7 +1210,7 @@ mod tests {
),
),
];
let p = plan_with(&c, &table(&t));
let p = plan_with(&LocalTree, &c, &table(&t));
assert_eq!(p.moves.len(), 2);
for m in &p.moves {
assert!(m.to.starts_with("PS4/CUSA00900 - Bloodborne/"), "{}", m.to);
@@ -1071,7 +1250,7 @@ mod tests {
),
),
];
let p = plan_with(&c, &table(&t));
let p = plan_with(&LocalTree, &c, &table(&t));
for m in &p.moves {
assert!(
m.to.starts_with("PS4/PPSA14251 - Mafia The Old Country/"),
@@ -1096,7 +1275,7 @@ mod tests {
"1.32",
);
let t = [("one", i.clone()), ("two", i.clone()), ("three", i)];
let p = plan_with(&c, &table(&t));
let p = plan_with(&LocalTree, &c, &table(&t));
assert_eq!(p.moves.len(), 3);
assert_eq!(p.moves.iter().filter(|m| !m.duplicate).count(), 1);
let tos: HashSet<&str> = p.moves.iter().map(|m| m.to.as_str()).collect();
@@ -1121,7 +1300,7 @@ mod tests {
"patch",
"1.32",
);
let p = plan_with(&c, &table(&[("one", i.clone()), ("two", i)]));
let p = plan_with(&LocalTree, &c, &table(&[("one", i.clone()), ("two", i)]));
assert_eq!(p.moves.len(), 2);
assert!(p.moves.iter().all(|m| !m.duplicate));
assert_ne!(p.moves[0].to, p.moves[1].to);
@@ -1145,6 +1324,7 @@ mod tests {
..PkgInfo::default()
};
let p = plan_with(
&LocalTree,
&c,
&table(&[("partial", partial), ("broken", broken), ("noid", noid)]),
);
@@ -1162,14 +1342,15 @@ mod tests {
let root = l.0.clone();
put(&root, "game.pkg", 10);
put(&root.join("fpkg"), "other.pkg", 10);
let found = containers(&root);
let found = containers(&LocalTree, &root);
assert!(found.contains(&root), "{found:?}");
assert!(found.contains(&root.join("fpkg")), "{found:?}");
// The root plans only its loose package; the subfolder is a container of its own.
assert_eq!(scan_top(&root), vec![root.join("game.pkg")]);
assert_eq!(scan_top(&LocalTree, &root), vec![root.join("game.pkg")]);
let p = plan_files(
&LocalTree,
&root,
scan_top(&root),
scan_top(&LocalTree, &root),
&table(&[(
"game",
info(
@@ -1198,7 +1379,7 @@ mod tests {
"1.02",
);
i.kind_confident = false;
let p = plan_with(&c, &table(&[("u", i)]));
let p = plan_with(&LocalTree, &c, &table(&[("u", i)]));
assert!(
p.moves[0].to.ends_with("PATCH_UNSURE.pkg"),
"{}",
@@ -1220,7 +1401,7 @@ mod tests {
"base",
"1.00",
);
let p = plan_with(&c, &table(&[("Big", i)]));
let p = plan_with(&LocalTree, &c, &table(&[("Big", i)]));
let names: Vec<&str> = p
.moves
.iter()
@@ -1245,25 +1426,25 @@ mod tests {
let rdr2 = info("RDR2", "CUSA03041", "UP1004-CUSA03041_00-X", "base", "1.00");
// Found by its old name before the move and its new one after.
let t = [("x", rdr2.clone()), ("RDR2", rdr2)];
let first = plan_with(&c, &table(&t));
let first = plan_with(&LocalTree, &c, &table(&t));
assert_eq!(first.moves.len(), 1);
let log_path = l.0.join("logs/run.json");
let log = apply(&c, first.moves.clone(), &log_path).unwrap();
let log = apply(&LocalTree, &c, first.moves.clone(), &log_path, "", None).unwrap();
assert_eq!(log.completed.len(), 1);
assert!(c.join(&first.moves[0].to).is_file());
// The emptied folders (only Finder's files left) are gone; other files stay.
assert!(!c.join("loose").exists(), "{:?}", log.kept_dirs);
assert!(c.join("Note.txt").is_file());
let again = plan_with(&c, &table(&t));
let again = plan_with(&LocalTree, &c, &table(&t));
assert!(again.moves.is_empty());
assert_eq!(again.skipped[0].reason, SKIP_IN_PLACE);
let undone = revert(&log_path).unwrap();
let undone = revert(&log_path, &local).unwrap();
assert_eq!(undone.reverted.len(), 1);
assert!(c.join("loose/inner/x.pkg").is_file());
assert!(!c.join("PS4").exists());
assert!(revert(&log_path).is_err(), "a run is undone once");
assert!(revert(&log_path, &local).is_err(), "a run is undone once");
}
#[test]
@@ -1275,9 +1456,12 @@ mod tests {
serde_json::from_value(serde_json::json!({ "from": from, "to": to })).unwrap()
};
let log = apply(
&LocalTree,
&c,
vec![mv("x.pkg", "../out.pkg"), mv("/etc/hosts", "y.pkg")],
&l.0.join("log.json"),
"",
None,
)
.unwrap();
assert_eq!(log.failed.len(), 2);
@@ -1298,15 +1482,15 @@ mod tests {
"x",
info("RDR2", "CUSA03041", "UP1004-CUSA03041_00-X", "base", "1.00"),
)];
let p = plan_with(&c, &table(&t));
let p = plan_with(&LocalTree, &c, &table(&t));
let log_path = l.0.join("log.json");
let log = apply(&c, p.moves.clone(), &log_path).unwrap();
let log = apply(&LocalTree, &c, p.moves.clone(), &log_path, "", None).unwrap();
assert_eq!(log.completed.len(), 1);
// As if the engine died before the final write: the log knows only the plan.
let mut stale = log.clone();
stale.completed.clear();
write_log(&log_path, &stale).unwrap();
let undone = revert(&log_path).unwrap();
let undone = revert(&log_path, &local).unwrap();
assert_eq!(undone.reverted.len(), 1);
assert!(c.join("x.pkg").is_file());
}
@@ -1320,7 +1504,7 @@ mod tests {
"x",
info("RDR2", "CUSA03041", "UP1004-CUSA03041_00-X", "base", "1.00"),
)];
let p = plan_with(&c, &table(&t));
let p = plan_with(&LocalTree, &c, &table(&t));
put(
&c.join(Path::new(&p.moves[0].to).parent().unwrap()),
Path::new(&p.moves[0].to)
@@ -1330,7 +1514,7 @@ mod tests {
.unwrap(),
99,
);
let log = apply(&c, p.moves, &l.0.join("log.json")).unwrap();
let log = apply(&LocalTree, &c, p.moves, &l.0.join("log.json"), "", None).unwrap();
assert_eq!(log.failed.len(), 1);
assert!(c.join("x.pkg").is_file());
}
@@ -1347,14 +1531,14 @@ mod tests {
"base",
"1.00",
);
let p = plan_with(&c, &table(&[]));
let p = plan_with(&LocalTree, &c, &table(&[]));
assert!(p.moves.is_empty());
let name = format!("{} - PPSA00001 - v1.00 - US - BASE.pkg", title)
.nfd()
.collect::<String>();
let folder = format!("PPSA00001 - {title}").nfd().collect::<String>();
put(&c.join("PS4").join(&folder), &name, 10);
let p = plan_with(&c, &table(&[("Ghost", i)]));
let p = plan_with(&LocalTree, &c, &table(&[("Ghost", i)]));
assert!(
p.moves.is_empty(),
"{:?}",
@@ -0,0 +1,910 @@
//! Where the organizer's files are: this computer's disk, or a folder on a saved server.
//!
//! The organizer only ever lists, looks up, makes folders, renames without replacing, and
//! removes folders it emptied. [`Tree`] is that, for both. Paths are `PathBuf`s holding what
//! the Collection holds: a local path, or a `remote://<connection>/<path>` string (the server
//! form is joined and compared like any path, and parsed back only at the server's door).
//!
//! A server tree blocks on the runtime, so it is used from `spawn_blocking`, never from a
//! runtime worker. Its listings are kept for the life of the tree (one request) and dropped on
//! every write, so a plan costs one round trip per folder and an apply's checks see the server
//! as it is now.
use std::collections::HashMap;
use std::path::Path;
use std::sync::{Arc, Mutex};
use crate::remote::pool::{is_transient, Pool};
use crate::remote::store::Store;
use crate::remote::{RemoteError, RemoteFs};
/// One entry of a folder listing.
#[derive(Debug, Clone)]
pub struct DirEnt {
pub name: String,
pub is_dir: bool,
/// A regular file (a symlink on this computer is neither file nor folder).
pub is_file: bool,
pub size: u64,
/// Seconds since the epoch; 0 when unknown.
pub mtime: i64,
}
#[derive(Debug, Clone, Copy)]
pub struct Meta {
pub size: u64,
pub mtime: i64,
}
#[derive(Debug, Clone, PartialEq)]
pub enum OpError {
/// The name is taken; nothing was changed.
Exists(String),
/// The server stopped answering and a fresh sign-in did not help. What the last operation
/// did is unknown until the server can be asked again.
Lost(String),
Other(String),
}
impl std::fmt::Display for OpError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
OpError::Exists(m) | OpError::Lost(m) | OpError::Other(m) => f.write_str(m),
}
}
}
pub trait Tree: Send + Sync {
/// The entries of `dir`, or `None` when it cannot be read.
fn list(&self, dir: &Path) -> Option<Vec<DirEnt>>;
/// A file's size and time; may come from the listing of its folder.
fn meta(&self, p: &Path) -> Option<Meta>;
/// Whether anything is at `p` now (never from a cached listing).
fn exists(&self, p: &Path) -> Result<bool, OpError>;
/// One folder; one already there is success.
fn create_dir(&self, p: &Path) -> Result<(), OpError>;
/// Renames without ever replacing what is at `to`.
fn rename_new(&self, from: &Path, to: &Path) -> Result<(), OpError>;
/// Removes an empty folder.
fn remove_dir(&self, p: &Path) -> Result<(), OpError>;
/// A file of Finder's that may be deleted to empty a folder. Never on a server: the
/// organizer deletes no file there.
fn is_removable_junk(&self, p: &Path) -> bool;
fn remove_junk(&self, p: &Path);
/// Something that went wrong reading the tree (a folder that could not be listed), for
/// the preview to say so rather than show a quietly smaller plan.
fn trouble(&self) -> Option<String> {
None
}
}
// ─── This computer ─────────────────────────────────────────────────────
pub struct LocalTree;
fn secs(t: std::io::Result<std::time::SystemTime>) -> i64 {
t.ok()
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn io(e: std::io::Error) -> OpError {
if e.kind() == std::io::ErrorKind::AlreadyExists {
OpError::Exists(e.to_string())
} else {
OpError::Other(e.to_string())
}
}
impl Tree for LocalTree {
fn list(&self, dir: &Path) -> Option<Vec<DirEnt>> {
let rd = std::fs::read_dir(dir).ok()?;
Some(
rd.flatten()
.map(|e| {
let ft = e.file_type().ok();
let m = e.metadata().ok();
DirEnt {
name: e.file_name().to_string_lossy().into_owned(),
is_dir: ft.is_some_and(|t| t.is_dir()),
is_file: ft.is_some_and(|t| t.is_file()),
size: m.as_ref().map(|m| m.len()).unwrap_or(0),
mtime: m.map(|m| secs(m.modified())).unwrap_or(0),
}
})
.collect(),
)
}
fn meta(&self, p: &Path) -> Option<Meta> {
let m = std::fs::metadata(p).ok()?;
Some(Meta {
size: m.len(),
mtime: secs(m.modified()),
})
}
fn exists(&self, p: &Path) -> Result<bool, OpError> {
match std::fs::symlink_metadata(p) {
Ok(_) => Ok(true),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(io(e)),
}
}
fn create_dir(&self, p: &Path) -> Result<(), OpError> {
match std::fs::create_dir(p) {
Ok(()) => Ok(()),
Err(_) if p.is_dir() => Ok(()),
Err(e) => Err(io(e)),
}
}
fn rename_new(&self, from: &Path, to: &Path) -> Result<(), OpError> {
if self.exists(to)? {
return Err(OpError::Exists(format!("{} already exists", to.display())));
}
// A rename, never a copy: a move across volumes fails instead of copying.
std::fs::rename(from, to).map_err(io)
}
fn remove_dir(&self, p: &Path) -> Result<(), OpError> {
std::fs::remove_dir(p).map_err(io)
}
fn is_removable_junk(&self, p: &Path) -> bool {
super::junk::is_removable_junk(p)
}
fn remove_junk(&self, p: &Path) {
if super::junk::is_removable_junk(p) {
let _ = std::fs::remove_file(p);
}
}
}
// ─── A saved server ────────────────────────────────────────────────────
pub struct RemoteTree {
handle: tokio::runtime::Handle,
pool: Arc<Pool>,
store: Arc<Store>,
id: String,
/// Server path → its listing, until the next write.
cache: Mutex<HashMap<String, Vec<DirEnt>>>,
trouble: Mutex<Option<String>>,
}
impl RemoteTree {
/// A tree on connection `id`. Call from a blocking thread inside the runtime.
pub fn new(
handle: tokio::runtime::Handle,
pool: Arc<Pool>,
store: Arc<Store>,
id: &str,
) -> Self {
Self {
handle,
pool,
store,
id: id.to_string(),
cache: Mutex::new(HashMap::new()),
trouble: Mutex::new(None),
}
}
/// The tree for the server a `remote://` path is on, through the engine's pool.
pub fn for_path(path: &str) -> Result<Self, String> {
let (id, _) = split_raw(path).ok_or_else(|| format!("not a server path: {path}"))?;
let r = crate::remote::pool::global().map_err(|e| e.to_string())?;
let handle = tokio::runtime::Handle::try_current()
.map_err(|_| format!("{path}: no runtime to reach the server from"))?;
Ok(Self::new(
handle,
Arc::clone(&r.pool),
Arc::clone(&r.store),
&id,
))
}
/// The server path in a `remote://` path on this tree's connection.
fn server(&self, p: &Path) -> Result<String, OpError> {
let s = path_string(p);
let (id, path) =
split_raw(&s).ok_or_else(|| OpError::Other(format!("not a server path: {s}")))?;
if id != self.id {
return Err(OpError::Other(format!("{s} is on another server")));
}
Ok(path)
}
fn forget(&self) {
self.cache.lock().unwrap_or_else(|e| e.into_inner()).clear();
}
/// `f` on the session, once more on a fresh sign-in if the connection dropped. Only for
/// what is safe to repeat.
fn run<T, F, Fut>(&self, f: F) -> Result<T, RemoteError>
where
F: Fn(Arc<dyn RemoteFs>) -> Fut,
Fut: std::future::Future<Output = Result<T, RemoteError>>,
{
self.handle
.block_on(self.pool.with_fs(&self.store, &self.id, f))
}
fn list_server(&self, dir: &str) -> Result<Vec<DirEnt>, RemoteError> {
if let Some(v) = self
.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.get(dir)
{
return Ok(v.clone());
}
let mut out = Vec::new();
let mut cursor = None;
loop {
let page = self.run(|fs| {
let (dir, cursor) = (dir.to_string(), cursor.clone());
async move { fs.list(&dir, cursor).await }
})?;
out.extend(page.entries.into_iter().map(|e| DirEnt {
is_file: !e.is_dir,
name: e.name,
is_dir: e.is_dir,
size: e.size,
mtime: e.mtime.unwrap_or(0),
}));
match page.next_cursor {
Some(c) => cursor = Some(c),
None => break,
}
}
if cfg!(windows) {
// A path here is a `PathBuf`, whose separator on Windows is also `\`: a name with
// one in it could not be told from two names, so it is left out (never touched).
out.retain(|e| !e.name.contains('\\'));
}
self.cache
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(dir.to_string(), out.clone());
Ok(out)
}
fn exists_server(&self, p: &str) -> Result<bool, OpError> {
match self.run(|fs| {
let p = p.to_string();
async move { fs.stat(&p).await }
}) {
Ok(_) => Ok(true),
Err(RemoteError::NotFound(_)) => Ok(false),
Err(e) => Err(op(e)),
}
}
/// A write's error, told apart: an error that looks like a dropped connection is
/// [`OpError::Lost`] only when the server does not answer a look at its root either (a
/// server can fail one request, an HTTP 500 say, and be fine).
fn classify(&self, e: RemoteError) -> OpError {
match op(e) {
OpError::Lost(m) if self.exists_server("/").is_ok() => OpError::Other(m),
other => other,
}
}
fn rename_once(&self, from: &str, to: &str) -> Result<(), RemoteError> {
self.handle.block_on(async {
let fs = self.pool.fs(&self.store, &self.id).await?;
fs.rename(from, to).await
})
}
}
fn op(e: RemoteError) -> OpError {
match e {
RemoteError::Exists(m) => OpError::Exists(m),
e if is_transient(&e) => OpError::Lost(e.to_string()),
e => OpError::Other(e.to_string()),
}
}
impl Tree for RemoteTree {
fn list(&self, dir: &Path) -> Option<Vec<DirEnt>> {
let d = self.server(dir).ok()?;
match self.list_server(&d) {
Ok(v) => Some(v),
Err(RemoteError::NotFound(_)) => None,
Err(e) => {
let mut t = self.trouble.lock().unwrap_or_else(|e| e.into_inner());
if t.is_none() {
*t = Some(format!("{d} could not be listed: {e}"));
}
None
}
}
}
fn meta(&self, p: &Path) -> Option<Meta> {
let s = self.server(p).ok()?;
let (dir, name) = s.rsplit_once('/')?;
let dir = if dir.is_empty() { "/" } else { dir };
if let Ok(list) = self.list_server(dir) {
return list.iter().find(|e| e.name == name).map(|e| Meta {
size: e.size,
mtime: e.mtime,
});
}
let e = self
.run(|fs| {
let s = s.clone();
async move { fs.stat(&s).await }
})
.ok()?;
Some(Meta {
size: e.size,
mtime: e.mtime.unwrap_or(0),
})
}
fn exists(&self, p: &Path) -> Result<bool, OpError> {
self.exists_server(&self.server(p)?)
}
fn create_dir(&self, p: &Path) -> Result<(), OpError> {
let s = self.server(p)?;
self.forget();
// Safe to repeat: a folder already there is success.
self.run(|fs| {
let s = s.clone();
async move { fs.create_dir(&s).await }
})
.map_err(|e| self.classify(e))
}
/// A rename is not safe to repeat blindly: the first try may have happened with its reply
/// lost. After a dropped connection the server is asked (on a fresh sign-in) where the
/// file is: at the new name only means done; at the old name only, it is tried once more;
/// anything else, or no answer, is [`OpError::Lost`].
fn rename_new(&self, from: &Path, to: &Path) -> Result<(), OpError> {
let (f, t) = (self.server(from)?, self.server(to)?);
self.forget();
let first = self.rename_once(&f, &t);
let e = match first {
Ok(()) => return Ok(()),
Err(e) if !is_transient(&e) => return Err(op(e)),
Err(e) => e,
};
self.pool.invalidate(&self.id);
let at_old = self.exists_server(&f);
let at_new = self.exists_server(&t);
self.forget();
match (at_old, at_new) {
(Ok(false), Ok(true)) => Ok(()),
(Ok(true), Ok(false)) => self.rename_once(&f, &t).map_err(|e| self.classify(e)),
(Err(OpError::Lost(_)), _) | (_, Err(OpError::Lost(_))) => {
Err(OpError::Lost(e.to_string()))
}
_ => Err(OpError::Other(e.to_string())),
}
}
fn remove_dir(&self, p: &Path) -> Result<(), OpError> {
let s = self.server(p)?;
self.forget();
match self.run(|fs| {
let s = s.clone();
async move { fs.remove_dir(&s).await }
}) {
Ok(()) => Ok(()),
// Gone either way (a repeat after a lost reply lands here).
Err(RemoteError::NotFound(_)) if self.exists_server(&s) == Ok(false) => Ok(()),
Err(e) => Err(self.classify(e)),
}
}
fn is_removable_junk(&self, _p: &Path) -> bool {
false
}
fn remove_junk(&self, _p: &Path) {}
fn trouble(&self) -> Option<String> {
self.trouble
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
}
}
/// A path as the organizer shows and stores it. On Windows a server path joined as a `PathBuf`
/// has `\` separators, which go back to `/`; anywhere else it is left as it is (on a server a
/// `\` is a name character).
pub fn path_string(p: &Path) -> String {
let s = p.to_string_lossy().into_owned();
if cfg!(windows) && crate::remote::path::is_remote(&s) {
s.replace('\\', "/")
} else {
s
}
}
/// `remote://<connection><server path>` in the organizer's form: the server path with its
/// names as the server gives them (never percent-decoded, so "a%20b.pkg" is that name).
/// Returns (connection id, server path), or `None` for anything else or a `.`/`..` segment.
pub fn split_raw(s: &str) -> Option<(String, String)> {
let rest = s.strip_prefix("remote://")?;
let (id, path) = rest.split_once('/').unwrap_or((rest, ""));
if id.is_empty()
|| !id
.bytes()
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
{
return None;
}
let mut segs = Vec::new();
for seg in path.split('/') {
match seg {
"" => continue,
"." | ".." => return None,
s if s.contains('\0') => return None,
s => segs.push(s),
}
}
Some((id.to_string(), format!("/{}", segs.join("/"))))
}
/// A collection root (as the settings hold it) in the organizer's form: a server root's path
/// decoded once into plain names.
pub fn raw_root(root: &str) -> std::path::PathBuf {
if crate::remote::path::is_remote(root) {
if let Ok(p) = crate::remote::path::parse(root) {
return std::path::PathBuf::from(format!("remote://{}{}", p.connection_id, p.path));
}
}
std::path::PathBuf::from(root)
}
/// The tree a collection path is in: a server's for a `remote://` path, else this computer.
pub fn tree_for(path: &Path) -> Result<Box<dyn Tree>, String> {
let s = path_string(path);
if crate::remote::path::is_remote(&s) {
Ok(Box::new(RemoteTree::for_path(&s)?))
} else {
Ok(Box::new(LocalTree))
}
}
#[cfg(test)]
mod tests {
use super::super::organize::{self, Facts};
use super::super::PkgInfo;
use super::*;
use crate::remote::pool::testing::remote_with_shared;
use crate::remote::store::{conn, Protocol, Secret};
use crate::remote::MemFs;
use std::path::PathBuf;
fn info(title: &str, tid: &str) -> PkgInfo {
PkgInfo {
platform: "PS4".into(),
title: title.into(),
title_id: tid.into(),
content_id: format!("UP0000-{tid}_00-AAAAAAAAAAAAAAAA"),
version: "1.00".into(),
region: "Americas".into(),
kind: "base".into(),
kind_confident: true,
complete: true,
..PkgInfo::default()
}
}
/// Facts by file-name prefix, read through the tree (sizes from the server's listing).
fn facts(tree: &dyn Tree, p: &Path) -> Facts {
let name = p.file_name().unwrap().to_string_lossy().into_owned();
let parts = organize::split_siblings(tree, p);
let size = parts
.iter()
.filter_map(|q| tree.meta(q))
.map(|m| m.size)
.sum();
let i = match () {
_ if name.starts_with("rdr") || name.starts_with("RDR2") => info("RDR2", "CUSA03041"),
_ if name.starts_with("bb") || name.starts_with("Bloodborne") => {
info("Bloodborne", "CUSA00900")
}
_ if name.starts_with("jak") || name.starts_with("Jak") => info("Jak X", "CUSA07842"),
_ => PkgInfo::default(),
};
Facts {
path: p.to_path_buf(),
parts,
size,
info: i,
}
}
struct Server {
mem: Arc<MemFs>,
pool: Arc<Pool>,
store: Arc<Store>,
id: String,
logs: PathBuf,
}
impl Drop for Server {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.logs);
}
}
fn server(name: &str, files: &[(&str, &[u8])]) -> Server {
let mem = Arc::new(MemFs::new(files));
let r = remote_with_shared(Arc::clone(&mem), None);
let id = r
.store
.add(conn("NAS", Protocol::Smb), Secret::None)
.unwrap()
.conn
.id;
let logs = std::env::temp_dir().join(format!("ps5u-rorg-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&logs);
Server {
mem,
pool: Arc::clone(&r.pool),
store: Arc::clone(&r.store),
id,
logs,
}
}
impl Server {
fn tree(&self) -> RemoteTree {
RemoteTree::new(
tokio::runtime::Handle::current(),
Arc::clone(&self.pool),
Arc::clone(&self.store),
&self.id,
)
}
fn path(&self, p: &str) -> PathBuf {
PathBuf::from(format!("remote://{}{p}", self.id))
}
}
/// Runs `f` where a server tree may block: off the runtime's workers.
async fn blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> T {
tokio::task::spawn_blocking(f).await.unwrap()
}
#[tokio::test(flavor = "multi_thread")]
async fn a_server_folder_is_planned_applied_and_undone() {
let s = Arc::new(server(
"cycle",
&[
("/games/fpkg/rdr.pkg", b"rdr2-bytes"),
("/games/fpkg/old/bb.pkg", b"bloodborne"),
("/games/fpkg/notes.txt", b"keep"),
],
));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/games/fpkg");
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
assert_eq!(plan.moves.len(), 2, "{:?}", plan.moves);
assert!(plan
.moves
.iter()
.all(|m| m.container == c.to_string_lossy() && m.to.starts_with("PS4/")));
let log_path = s.logs.join("run.json");
let log =
organize::apply(&tree, &c, plan.moves.clone(), &log_path, "NAS", None).unwrap();
assert_eq!(log.completed.len(), 2, "{:?}", log.failed);
assert_eq!(log.server, "NAS");
for m in &plan.moves {
let to = format!("/games/fpkg/{}", m.to);
assert!(s.mem.bytes(&to).is_some(), "{to}");
assert!(s.mem.bytes(&format!("/games/fpkg/{}", m.from)).is_none());
}
// The folder the run emptied is gone; other files stay.
assert!(log.removed_dirs.contains(&"old".to_string()), "{log:?}");
assert_eq!(s.mem.bytes("/games/fpkg/notes.txt").unwrap(), b"keep");
let again = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
assert!(again.moves.is_empty(), "{:?}", again.moves);
let id = s.id.clone();
let (pool, store) = (Arc::clone(&s.pool), Arc::clone(&s.store));
let open = move |_: &Path| -> Result<Box<dyn Tree>, String> {
Ok(Box::new(RemoteTree::new(
tokio::runtime::Handle::current(),
Arc::clone(&pool),
Arc::clone(&store),
&id,
)))
};
let undone = organize::revert(&log_path, &open).unwrap();
assert_eq!(undone.reverted.len(), 2, "{:?}", undone.problems);
assert_eq!(s.mem.bytes("/games/fpkg/rdr.pkg").unwrap(), b"rdr2-bytes");
assert_eq!(
s.mem.bytes("/games/fpkg/old/bb.pkg").unwrap(),
b"bloodborne"
);
// The folders the run made are gone again.
assert!(s.tree().list(&s.path("/games/fpkg/PS4")).is_none());
assert!(organize::revert(&log_path, &open).is_err(), "undone once");
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_name_taken_on_the_server_since_the_preview_is_never_overwritten() {
let s = Arc::new(server("taken", &[("/g/rdr.pkg", b"mine")]));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/g");
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
let to = format!("/g/{}", plan.moves[0].to);
// Someone else's file lands at the destination after the preview.
s.mem.add(&[(to.as_str(), b"theirs")]);
let log =
organize::apply(&tree, &c, plan.moves, &s.logs.join("l.json"), "", None).unwrap();
assert_eq!(log.completed.len(), 0);
assert_eq!(log.failed[0].error, "destination appeared");
assert_eq!(s.mem.bytes(&to).unwrap(), b"theirs");
assert_eq!(s.mem.bytes("/g/rdr.pkg").unwrap(), b"mine");
// And the tree's own rename refuses a taken name, leaving both.
let r = tree.rename_new(
&s.path("/g/rdr.pkg"),
&PathBuf::from(format!("remote://{}{to}", s.id)),
);
assert!(matches!(r, Err(OpError::Exists(_))), "{r:?}");
assert_eq!(s.mem.bytes(&to).unwrap(), b"theirs");
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_rename_whose_reply_was_lost_is_not_repeated() {
let s = Arc::new(server("lost-reply", &[("/g/a.pkg", b"a")]));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
s.mem.lose_next_replies(1);
tree.rename_new(&s.path("/g/a.pkg"), &s.path("/g/b.pkg"))
.expect("done, though its reply was lost");
assert_eq!(s.mem.bytes("/g/b.pkg").unwrap(), b"a");
assert!(s.mem.bytes("/g/a.pkg").is_none());
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_server_lost_mid_run_stops_it_and_undo_puts_back_what_happened() {
let s = Arc::new(server(
"down",
&[
("/g/bb.pkg", b"bloodborne"),
("/g/jak.pkg", b"jak"),
("/g/rdr.pkg", b"rdr2"),
],
));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/g");
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
assert_eq!(plan.moves.len(), 3);
// Move 1: make PS4, make its title folder, rename (3 writes). Move 2: make its
// title folder, then the rename happens but the server drops with its reply.
s.mem.go_down_after_writes(5);
let log_path = s.logs.join("down.json");
let log = organize::apply(&tree, &c, plan.moves.clone(), &log_path, "", None).unwrap();
assert_eq!(log.completed.len(), 1, "{log:?}");
assert_eq!(log.failed.len(), 2);
assert!(log.failed[0].uncertain, "{:?}", log.failed[0]);
assert!(!log.failed[1].uncertain);
assert!(log.failed[1].error.starts_with("not tried"));
// What the log on disk says is what happened.
let saved: organize::UndoLog =
serde_json::from_slice(&std::fs::read(&log_path).unwrap()).unwrap();
assert_eq!(saved.completed.len(), 1);
// Undo while the server is still away: refused, nothing judged gone.
let open_tree = {
let (pool, store, id) = (Arc::clone(&s.pool), Arc::clone(&s.store), s.id.clone());
move |_: &Path| -> Result<Box<dyn Tree>, String> {
Ok(Box::new(RemoteTree::new(
tokio::runtime::Handle::current(),
Arc::clone(&pool),
Arc::clone(&store),
&id,
)))
}
};
assert!(organize::revert(&log_path, &open_tree).is_err());
// Back up: the completed move and the uncertain one (it did happen) go back.
s.mem.set_down(false);
let undone = organize::revert(&log_path, &open_tree).unwrap();
assert_eq!(undone.reverted.len(), 2, "{:?}", undone.problems);
for (p, b) in [
("/g/bb.pkg", &b"bloodborne"[..]),
("/g/jak.pkg", b"jak"),
("/g/rdr.pkg", b"rdr2"),
] {
assert_eq!(s.mem.bytes(p).as_deref(), Some(b), "{p}");
}
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn the_index_spares_reading_packages_over_the_network() {
let s = Arc::new(server(
"cached",
&[("/games/fpkg/x.pkg", b"not a real package")],
));
s.mem.set_mtime("/games/fpkg/x.pkg", 1_700_000_000);
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let root = s.path("/games");
let mut lib = super::super::Library::default();
lib.pkg_cache.insert(
format!("{}|fpkg/x.pkg", root.display()),
super::super::PkgCacheEntry {
size: 18,
mtime: 1_700_000_000,
pkg: info("RDR2", "CUSA03041"),
v: super::super::IDENTIFY_VERSION,
},
);
let c = root.join("fpkg");
let roots = [(root.clone(), root.display().to_string())];
let plan = organize::plan_cached(&tree, &c, Some(&lib), &roots);
assert_eq!(plan.moves.len(), 1, "{:?}", plan.skipped);
assert!(plan.moves[0].to.starts_with("PS4/CUSA03041 - RDR2/"));
// A changed file is read again rather than trusted.
s.mem.set_mtime("/games/fpkg/x.pkg", 1_700_000_001);
let tree = s.tree();
let plan = organize::plan_cached(&tree, &c, Some(&lib), &roots);
assert!(plan.moves.is_empty());
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_server_that_never_removes_folders_keeps_the_emptied_ones_and_says_so() {
let s = Arc::new(server("keep", &[("/g/old/rdr.pkg", b"rdr2")]));
s.mem.never_remove_dirs();
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/g");
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
let log =
organize::apply(&tree, &c, plan.moves, &s.logs.join("k.json"), "", None).unwrap();
assert_eq!(log.completed.len(), 1);
assert!(log.removed_dirs.is_empty());
assert_eq!(log.kept_dirs, vec!["old".to_string()]);
assert!(s
.tree()
.list(&s.path("/g/old"))
.is_some_and(|v| v.is_empty()));
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn odd_server_names_are_renamed_as_themselves() {
let s = Arc::new(server(
"names",
&[
("/g/rdr%20x.pkg", b"percent"),
("/g/rdr x.pkg", b"space"),
("/g/bb 100% .pkg", b"orange"),
("/g/jak\\b.pkg", b"backslash"),
],
));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/g");
for (name, bytes) in [
("rdr%20x.pkg", &b"percent"[..]),
("rdr x.pkg", b"space"),
("bb 100% .pkg", b"orange"),
("jak\\b.pkg", b"backslash"),
] {
let m = tree.meta(&c.join(name)).expect(name);
assert_eq!(m.size, bytes.len() as u64, "{name}");
}
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
let froms: Vec<&str> = plan.moves.iter().map(|m| m.from.as_str()).collect();
for name in ["rdr%20x.pkg", "rdr x.pkg", "bb 100% .pkg", "jak\\b.pkg"] {
assert!(
froms.contains(&name),
"{name} not planned as itself: {froms:?}"
);
}
let log = organize::apply(
&tree,
&c,
plan.moves.clone(),
&s.logs.join("n.json"),
"",
None,
)
.unwrap();
assert_eq!(log.completed.len(), 4, "{:?}", log.failed);
for m in &plan.moves {
let want: &[u8] = match m.from.as_str() {
"rdr%20x.pkg" => b"percent",
"rdr x.pkg" => b"space",
"bb 100% .pkg" => b"orange",
_ => b"backslash",
};
assert_eq!(
s.mem.bytes(&format!("/g/{}", m.to)).as_deref(),
Some(want),
"{}",
m.from
);
}
})
.await;
}
#[tokio::test(flavor = "multi_thread")]
async fn a_name_taken_in_another_case_is_not_landed_on() {
let s = Arc::new(server("case", &[("/g/rdr.pkg", b"mine")]));
let s2 = Arc::clone(&s);
blocking(move || {
let s = s2;
let tree = s.tree();
let c = s.path("/g");
let first = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
let to = first.moves[0].to.clone();
// The same name in another case, already there (a case-insensitive server's twin).
let shout = format!("/g/{}", to.to_uppercase());
s.mem.add(&[(shout.as_str(), b"theirs")]);
let tree = s.tree();
let plan = organize::plan_with(&tree, &c, &|p| facts(&tree, p));
assert_eq!(plan.moves.len(), 1);
assert_ne!(
plan.moves[0].to.to_lowercase(),
to.to_lowercase(),
"{plan:?}"
);
let log = organize::apply(
&tree,
&c,
plan.moves.clone(),
&s.logs.join("c.json"),
"",
None,
)
.unwrap();
assert_eq!(log.completed.len(), 1);
assert_eq!(s.mem.bytes(&shout).unwrap(), b"theirs");
assert_eq!(
s.mem.bytes(&format!("/g/{}", plan.moves[0].to)).unwrap(),
b"mine"
);
})
.await;
}
}
@@ -59,8 +59,6 @@ pub enum Fsys {
Local,
Remote {
fs: std::sync::Arc<crate::remote::source_fs::RemoteSourceFs>,
/// `remote://<connection-id>`.
base: String,
},
}
@@ -76,14 +74,11 @@ impl Fsys {
};
}
let rt = rt.ok_or_else(|| format!("{root}: no runtime to reach the server from"))?;
let p = crate::remote::path::parse(root).map_err(|e| e.to_string())?;
crate::remote::path::parse(root).map_err(|e| e.to_string())?;
let (fs, _) = rt
.block_on(crate::remote::source_fs::RemoteSourceFs::for_path(root))
.map_err(|e| format!("{root}: {e}"))?;
let f = Fsys::Remote {
fs,
base: format!("remote://{}", p.connection_id),
};
let f = Fsys::Remote { fs };
if f.try_list(root).is_none() {
return Err(format!("{root} could not be listed on the server"));
}
@@ -91,7 +86,6 @@ impl Fsys {
}
fn try_list(&self, dir: &str) -> Option<Vec<Ent>> {
use ps5upload_core::source_fs::SourceFs;
match self {
Fsys::Local => {
let rd = std::fs::read_dir(dir).ok()?;
@@ -123,25 +117,23 @@ impl Fsys {
.collect(),
)
}
Fsys::Remote { fs, base } => {
Fsys::Remote { fs, .. } => {
let server = crate::remote::path::parse(dir).ok()?.path;
let children = fs.read_dir(Path::new(&server)).ok()?;
let children = fs.entries(&server).ok()?;
// Each name encoded into the `remote://` path, so that reading it back gives
// this name ("a%20b.pkg" stays itself, a backslash stays a name character).
let at = dir.trim_end_matches('/');
Some(
children
.into_iter()
.map(|(child, is_dir)| {
let size = fs.metadata(&child).map(|m| m.len).unwrap_or(0);
let mtime = fs.mtime(&child).unwrap_or(0) as i64;
let sp = child.to_string_lossy().replace('\\', "/");
Ent {
name: sp.rsplit('/').next().unwrap_or(&sp).to_string(),
path: format!("{base}{sp}"),
is_dir,
is_symlink: false,
size,
mtime,
created: 0,
}
.map(|e| Ent {
path: format!("{at}/{}", crate::remote::path::encode_segment(&e.name)),
name: e.name,
is_dir: e.is_dir,
is_symlink: false,
size: e.size,
mtime: e.mtime.filter(|t| *t > 0).unwrap_or(0),
created: 0,
})
.collect(),
)
@@ -184,11 +176,13 @@ fn rar_parts(entries: &[Ent]) -> Vec<&Ent> {
/// multipart RAR set; otherwise it is only a place to look further. Symlinks are not followed.
pub fn discover(fs: &Fsys, root: &str, cancel: &AtomicBool) -> Vec<Found> {
let mut found = Vec::new();
fn rel_of(root: &str, full: &str) -> String {
full.strip_prefix(root)
.unwrap_or(full)
.trim_start_matches(['/', '\\'])
.replace('\\', "/")
fn rel_of(fs: &Fsys, root: &str, full: &str) -> String {
let rest = full.strip_prefix(root).unwrap_or(full);
match fs {
// Server paths are `/`-separated, their names encoded: the relative path is names.
Fsys::Remote { .. } => crate::remote::path::decode_rel(rest.trim_start_matches('/')),
Fsys::Local => rest.trim_start_matches(['/', '\\']).replace('\\', "/"),
}
}
fn walk(
fs: &Fsys,
@@ -205,7 +199,7 @@ pub fn discover(fs: &Fsys, root: &str, cancel: &AtomicBool) -> Vec<Found> {
if e.is_symlink {
continue;
}
let rel = rel_of(root, &e.path);
let rel = rel_of(fs, root, &e.path);
if e.is_dir {
let inner = fs.list(&e.path);
let stat = (e.size, e.mtime, e.created);
@@ -812,7 +806,6 @@ mod tests {
let root = format!("remote://{id}/games");
let fsys = Fsys::Remote {
fs: std::sync::Arc::new(fs),
base: format!("remote://{id}"),
};
let (found, sizes) = tokio::task::spawn_blocking(move || {
let found = discover(&fsys, &root, &AtomicBool::new(false));
@@ -852,6 +845,69 @@ mod tests {
assert_eq!(sizes, (102, 5));
}
/// Server names that look like escapes or separators are found, and read back, as
/// themselves: "a%20b.pkg" is not "a b.pkg", and a backslash is part of the name.
#[tokio::test(flavor = "multi_thread")]
async fn odd_server_names_resolve_to_themselves() {
use crate::remote::pool::testing::remote_with_shared;
use crate::remote::store::{conn, Protocol, Secret};
use crate::remote::MemFs;
use ps5upload_core::source_fs::SourceFs;
let files: [(&str, &[u8]); 4] = [
("/games/a%20b.pkg", b"percent"),
("/games/a b.pkg", b"space"),
("/games/100% Orange.pkg", b"orange"),
("/games/a\\b.pkg", b"backslash"),
];
let r = remote_with_shared(std::sync::Arc::new(MemFs::new(&files)), None);
let id = r
.store
.add(conn("NAS", Protocol::Smb), Secret::None)
.unwrap()
.conn
.id;
let fs = std::sync::Arc::new(
crate::remote::source_fs::RemoteSourceFs::new(
std::sync::Arc::clone(&r.pool),
std::sync::Arc::clone(&r.store),
&id,
)
.await
.unwrap(),
);
let root = format!("remote://{id}/games");
let fsys = Fsys::Remote {
fs: std::sync::Arc::clone(&fs),
};
let found = tokio::task::spawn_blocking(move || {
let found = discover(&fsys, &root, &AtomicBool::new(false));
found
.into_iter()
.map(|f| {
let server = crate::remote::path::parse(&f.abs).unwrap().path;
let mut bytes = Vec::new();
std::io::Read::read_to_end(
&mut fs.open(Path::new(&server)).unwrap(),
&mut bytes,
)
.unwrap();
(f.rel, server, bytes)
})
.collect::<Vec<_>>()
})
.await
.unwrap();
assert_eq!(found.len(), 4);
for (name, bytes) in files {
let rel = name.trim_start_matches("/games/");
let hit = found.iter().find(|(r, _, _)| r == rel).unwrap_or_else(|| {
panic!("{rel} not found as itself: {found:?}");
});
assert_eq!(hit.1, name, "{rel} read back as another path");
assert_eq!(hit.2, bytes, "{rel} read another file");
}
}
#[test]
fn a_new_scan_starts_its_progress_from_nothing() {
let p = Progress::default();
@@ -14,6 +14,7 @@ use std::time::{Duration, Instant};
use axum::{http::StatusCode, response::Response, Json};
use serde::{Deserialize, Serialize};
use crate::collection::organize_fs::{path_string, raw_root, split_raw, tree_for, Tree};
use crate::collection::{junk, organize, store};
use crate::collection_api::{err, library, settings};
@@ -295,43 +296,132 @@ fn pending_plans() -> &'static Mutex<HashMap<String, PendingPlan>> {
P.get_or_init(|| Mutex::new(HashMap::new()))
}
/// This computer's roots: the organizer renames files, which it does only here.
fn roots() -> Vec<PathBuf> {
/// Every collection root, on this computer or on a saved server: (the root as the organizer's
/// trees take it, the root as the settings and the index name it).
fn roots() -> Vec<(PathBuf, String)> {
settings()
.roots
.iter()
.filter(|r| !crate::remote::path::is_remote(r))
.map(PathBuf::from)
.filter(|r| !r.trim().is_empty())
.map(|r| (raw_root(r), r.clone()))
.collect()
}
/// Every collection folder holding packages, across the roots. The index already knows which
/// top-level folders hold packages; walking the roots to find out reads every file of every
/// game folder (measured: 189,000 files, a minute on an external drive) to learn nothing new.
fn all_containers() -> Vec<PathBuf> {
fn path_str(p: &FsPath) -> String {
path_string(p)
}
/// The saved connection a path is on, if it is on one.
fn connection_of(p: &FsPath) -> Option<String> {
split_raw(&path_str(p)).map(|(id, _)| id)
}
/// The saved connection with id `id`, as saved now.
fn saved(id: &str) -> Option<crate::remote::store::Connection> {
crate::remote::pool::global()
.ok()
.and_then(|r| r.store.get(id))
.map(|(c, _)| c)
}
/// The saved server's name for a `remote://` path ("" on this computer).
fn server_name(p: &FsPath) -> String {
let Some(id) = connection_of(p) else {
return String::new();
};
saved(&id).map(|c| c.name).unwrap_or(id)
}
/// How a folder is shown: its path here, or "Server: /path" on a saved server.
fn label(p: &FsPath) -> String {
let s = path_str(p);
match split_raw(&s) {
Some((_, path)) => format!("{}: {path}", server_name(p)),
None => s,
}
}
/// Where a server folder is, for the undo log: which server, and the folder on it.
fn origin_of(p: &FsPath) -> Option<organize::Origin> {
let (id, path) = split_raw(&path_str(p))?;
let c = saved(&id)?;
Some(organize::Origin {
protocol: format!("{:?}", c.protocol).to_ascii_lowercase(),
host: c.host.trim().to_ascii_lowercase(),
port: c.port,
share: c.share.trim_matches(['/', '\\']).to_ascii_lowercase(),
path,
})
}
/// One [`Tree`] per place (this computer, or each server) for one request, so a server's
/// folders are listed once however many containers share it.
#[derive(Default)]
struct Trees(HashMap<String, Result<Box<dyn Tree>, String>>);
impl Trees {
fn get(&mut self, p: &FsPath) -> Result<&dyn Tree, String> {
let k = connection_of(p).unwrap_or_default();
self.0
.entry(k)
.or_insert_with(|| tree_for(p))
.as_ref()
.map(|t| t.as_ref())
.map_err(Clone::clone)
}
}
/// Every collection folder holding packages, across the roots, with what could not be
/// reached. The index already knows which top-level folders hold packages; walking the roots
/// to find out reads every file of every game folder (measured: 189,000 files, a minute on an
/// external drive) to learn nothing new.
fn all_containers(trees: &mut Trees) -> (Vec<PathBuf>, Vec<String>) {
let roots = roots();
let mut problems = Vec::new();
let Some(lib) = library() else {
return roots.iter().flat_map(|r| organize::containers(r)).collect();
let mut out = Vec::new();
for (r, _) in &roots {
match trees.get(r) {
Ok(t) => out.extend(organize::containers(t, r)),
Err(e) => problems.push(format!("{}: {e}", label(r))),
}
}
return (out, problems);
};
// A package lying loose in a root has no container: the root itself is its folder.
let mut out: Vec<PathBuf> = lib
let mut wanted: Vec<PathBuf> = lib
.games
.values()
.flat_map(|g| g.locations.iter())
.filter(|l| l.kind == "pkg")
.filter(|l| roots.iter().any(|r| r == FsPath::new(&l.root)))
.map(|l| {
if l.container.is_empty() {
PathBuf::from(&l.root)
.filter_map(|l| {
let (root, _) = roots.iter().find(|(_, key)| *key == l.root)?;
Some(if l.container.is_empty() {
root.clone()
} else {
FsPath::new(&l.root).join(&l.container)
}
root.join(&l.container)
})
})
.filter(|c| c.is_dir())
.collect();
out.sort();
out.dedup();
out
wanted.sort();
wanted.dedup();
let mut out = Vec::new();
for c in wanted {
let here = trees.get(&c).and_then(|t| {
t.list(&c)
.map(|_| ())
.ok_or_else(|| t.trouble().unwrap_or_else(|| "not found".into()))
});
match here {
Ok(()) => out.push(c),
Err(e) if connection_of(&c).is_some() => {
problems.push(format!("{}: {e}", label(&c)));
}
// A local folder that is gone is simply not organized (the next scan drops it).
Err(_) => {}
}
}
(out, problems)
}
fn organize_dir() -> Option<PathBuf> {
@@ -342,6 +432,8 @@ fn organize_dir() -> Option<PathBuf> {
pub struct PlanResponse {
pub token: String,
pub containers: Vec<String>,
/// How to show each container: its path, or "Server: /path" on a saved server.
pub labels: std::collections::BTreeMap<String, String>,
pub moves: Vec<organize::Move>,
pub skipped: Vec<organize::Skipped>,
pub unsure_count: usize,
@@ -350,37 +442,50 @@ pub struct PlanResponse {
/// POST /api/collection/organize/plan: what the organizer would do. Touches nothing.
pub async fn post_organize_plan() -> Response {
// Off the async workers: it stats every package folder.
let containers = tokio::task::spawn_blocking(all_containers)
.await
.unwrap_or_default();
if containers.is_empty() {
return err(
StatusCode::BAD_REQUEST,
"no folder in the collection holds packages",
);
}
let plans = tokio::task::spawn_blocking({
let containers = containers.clone();
move || {
let lib = library();
let roots = roots();
containers
.iter()
.map(|c| organize::plan_cached(c, lib.as_deref(), &roots))
.collect::<Vec<_>>()
}
// Off the async workers: it lists every package folder, perhaps over the network.
let (containers, problems, plans) = tokio::task::spawn_blocking(|| {
let mut trees = Trees::default();
let (containers, problems) = all_containers(&mut trees);
let lib = library();
let roots = roots();
let plans = containers
.iter()
.map(|c| match trees.get(c) {
Ok(t) => organize::plan_cached(t, c, lib.as_deref(), &roots),
Err(_) => organize::Plan::default(),
})
.collect::<Vec<_>>();
(containers, problems, plans)
})
.await
.unwrap_or_default();
if containers.is_empty() {
return err(
StatusCode::BAD_REQUEST,
if problems.is_empty() {
"no folder in the collection holds packages".to_string()
} else {
format!(
"no folder with packages could be read: {}",
problems.join("; ")
)
},
);
}
let mut resp = PlanResponse {
token: new_token(),
containers: containers
.iter()
.map(|c| c.to_string_lossy().into_owned())
.collect(),
containers: containers.iter().map(|c| path_str(c)).collect(),
labels: containers.iter().map(|c| (path_str(c), label(c))).collect(),
moves: Vec::new(),
skipped: Vec::new(),
skipped: problems
.into_iter()
.map(|detail| organize::Skipped {
container: String::new(),
path: String::new(),
reason: organize::SKIP_UNREADABLE.into(),
detail,
})
.collect(),
unsure_count: 0,
total_bytes: 0,
};
@@ -518,11 +623,10 @@ pub async fn post_organize_apply(Json(req): Json<OrganizeApplyReq>) -> Response
"no data folder for the undo log",
);
};
// Off the async workers: it stats every package folder.
let containers = tokio::task::spawn_blocking(all_containers)
.await
.unwrap_or_default();
// Off the async workers: it lists every package folder, perhaps over the network.
let result = tokio::task::spawn_blocking(move || {
let mut trees = Trees::default();
let (containers, _) = all_containers(&mut trees);
let lib = library();
let roots = roots();
let mut out = OrganizeApplyResult::default();
@@ -537,7 +641,14 @@ pub async fn post_organize_apply(Json(req): Json<OrganizeApplyReq>) -> Response
out.dropped.extend(chosen.into_iter().map(|m| m.from));
continue;
}
let plan = organize::plan_cached(&c, lib.as_deref(), &roots);
let tree = match trees.get(&c) {
Ok(t) => t,
Err(e) => {
out.dropped.push(format!("{}: {e}", label(&c)));
continue;
}
};
let plan = organize::plan_cached(tree, &c, lib.as_deref(), &roots);
let live: HashMap<&str, &organize::Move> =
plan.moves.iter().map(|m| (m.from.as_str(), m)).collect();
let mut picked = Vec::new();
@@ -567,7 +678,7 @@ pub async fn post_organize_apply(Json(req): Json<OrganizeApplyReq>) -> Response
)),
&plan.unsure,
);
match organize::apply(&c, picked, &log_path) {
match organize::apply(tree, &c, picked, &log_path, &server_name(&c), origin_of(&c)) {
Ok(log) => {
crate::engine_log::record(
"info",
@@ -581,7 +692,10 @@ pub async fn post_organize_apply(Json(req): Json<OrganizeApplyReq>) -> Response
);
out.moved.extend(log.completed);
out.failed.extend(log.failed);
out.kept_dirs.extend(log.kept_dirs);
// Where each is: one name ("old") could be in several containers.
let at = label(&c);
out.kept_dirs
.extend(log.kept_dirs.iter().map(|d| format!("{at}/{d}")));
out.logs.push(
log_path
.file_name()
@@ -609,6 +723,8 @@ pub async fn post_organize_apply(Json(req): Json<OrganizeApplyReq>) -> Response
pub struct RunSummary {
pub id: String,
pub container_root: String,
/// How to show the folder: its path, or "Server: /path" on a saved server.
pub label: String,
pub created_at: String,
pub moved: usize,
pub failed: usize,
@@ -631,8 +747,31 @@ pub async fn get_organize_runs() -> Response {
}
let log: organize::UndoLog =
serde_json::from_slice(&std::fs::read(e.path()).ok()?).ok()?;
let root = FsPath::new(&log.container_root);
let shown = match (
connection_of(root),
crate::remote::path::parse(&log.container_root),
) {
(Some(id), Ok(r)) => {
// The server's name now; the one it had then, should it since be gone.
let name = crate::remote::pool::global()
.ok()
.and_then(|g| g.store.get(&id))
.map(|(c, _)| c.name)
.unwrap_or_else(|| {
if log.server.is_empty() {
id.clone()
} else {
log.server.clone()
}
});
format!("{name}: {}", r.path)
}
_ => log.container_root.clone(),
};
Some(RunSummary {
id: name,
label: shown,
container_root: log.container_root,
created_at: log.created_at,
moved: log.completed.len(),
@@ -645,6 +784,36 @@ pub async fn get_organize_runs() -> Response {
ok(&runs)
}
/// Whether a run's folder is still where the run was made: the saved connection it names must
/// still point at the same server. An older log, from before that was recorded, is undone on
/// the connection with the same id (Undo itself checks the folder is there), with a note.
fn same_place(log_path: &FsPath) -> Result<Option<String>, String> {
let bytes = std::fs::read(log_path).map_err(|e| e.to_string())?;
let log: organize::UndoLog = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?;
let root = FsPath::new(&log.container_root);
let Some((id, _)) = split_raw(&log.container_root) else {
return Ok(None);
};
let Some(now) = origin_of(root) else {
return Err(format!(
"the saved connection this run was made on ({id}) is gone"
));
};
match &log.origin {
Some(then) if then.same_server(&now) => Ok(None),
Some(then) => Err(format!(
"the saved connection now points to another server ({} {}:{} instead of {} {}:{}); \
undo refused so nothing is renamed there",
now.protocol, now.host, now.port, then.protocol, then.host, then.port
)),
None => Ok(Some(format!(
"This run was logged before ps5upload recorded which server it was on: it was undone \
on the saved connection \"{}\" as it is set up now.",
server_name(root)
))),
}
}
#[derive(Debug, Deserialize)]
pub struct RevertReq {
pub id: String,
@@ -662,10 +831,16 @@ pub async fn post_organize_revert(Json(req): Json<RevertReq>) -> Response {
let Some(path) = organize_dir().map(|d| d.join(&req.id)) else {
return err(StatusCode::NOT_FOUND, "no such run");
};
let res = tokio::task::spawn_blocking(move || organize::revert(&path))
.await
.map_err(|e| std::io::Error::other(e.to_string()))
.and_then(|r| r);
let res = tokio::task::spawn_blocking(move || {
let note = same_place(&path).map_err(std::io::Error::other)?;
organize::revert(&path, &tree_for).map(|mut r| {
r.note = note;
r
})
})
.await
.map_err(|e| std::io::Error::other(e.to_string()))
.and_then(|r| r);
match res {
Ok(r) => {
crate::engine_log::record(
@@ -1094,6 +1269,50 @@ mod tests {
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn undo_is_refused_when_the_saved_connection_now_points_elsewhere() {
use crate::remote::store::{conn, Protocol, Secret};
let r = crate::remote::pool::testing::install_global(&[]);
let id = r
.store
.add(conn("NAS", Protocol::Smb), Secret::None)
.unwrap()
.conn
.id;
let root = format!("remote://{id}/games/fpkg");
let now = origin_of(FsPath::new(&root)).unwrap();
let d = std::env::temp_dir().join(format!("ps5u-place-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
let log = |origin: Option<organize::Origin>, root: &str| {
let p = d.join(format!("organize_{}.json", uuid::Uuid::new_v4()));
let v = serde_json::json!({
"version": 1, "container_root": root, "created_at": "x", "moves": [],
"origin": origin,
});
std::fs::write(&p, serde_json::to_vec(&v).unwrap()).unwrap();
p
};
assert_eq!(same_place(&log(Some(now.clone()), &root)), Ok(None));
let moved = organize::Origin {
host: "another-nas".into(),
..now.clone()
};
let refused = same_place(&log(Some(moved), &root)).unwrap_err();
assert!(refused.contains("another server"), "{refused}");
let other_share = organize::Origin {
share: "elsewhere".into(),
..now
};
assert!(same_place(&log(Some(other_share), &root)).is_err());
// An older log: undone on the same connection id, with a note.
assert!(same_place(&log(None, &root)).unwrap().is_some());
// A connection that is gone: refused.
assert!(same_place(&log(None, "remote://gone-0000/games")).is_err());
// This computer's folders have nothing to compare.
assert_eq!(same_place(&log(None, "/Volumes/x/fpkg")), Ok(None));
let _ = std::fs::remove_dir_all(&d);
}
#[test]
fn log_slugs_are_plain() {
assert_eq!(slug(FsPath::new("/x/game fpkgs (old)")), "game-fpkgs--old");
@@ -48,3 +48,76 @@ pub(crate) async fn check(fs: &dyn RemoteFs, root: &str) {
Err(RemoteError::NotFound(_))
));
}
/// What every writable [`RemoteFs`] must do. `dir` holds two files `a` and `b` of different
/// sizes; the tree is left as found, except that a backend which never removes folders
/// (WebDAV: its DELETE is recursive) leaves the test's empty folders behind.
pub(crate) async fn check_writes(fs: &dyn RemoteFs, dir: &str) {
let run = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let at = |n: &str| super::path::join(dir, &format!("{n}-{run}")).unwrap();
let file = |n: &str| super::path::join(dir, n).unwrap();
let (a, b) = (file("a"), file("b"));
let size_a = fs.stat(&a).await.expect("stat a").size;
let size_b = fs.stat(&b).await.expect("stat b").size;
assert_ne!(size_a, size_b, "a and b must differ in size");
// Never overwrite: the rename fails and both files stay exactly as they were.
let clash = fs.rename(&a, &b).await;
assert!(
matches!(clash, Err(RemoteError::Exists(_))),
"rename onto an existing file must fail as Exists, got {clash:?}"
);
assert_eq!(fs.stat(&a).await.expect("a survives").size, size_a);
assert_eq!(fs.stat(&b).await.expect("b survives").size, size_b);
// A folder: made once, made again without complaint; a file's name is not a folder.
let new = at("ps5u-new");
fs.create_dir(&new).await.expect("create a folder");
fs.create_dir(&new)
.await
.expect("a folder already there is fine");
assert!(fs.stat(&new).await.expect("stat the new folder").is_dir);
assert!(
fs.create_dir(&a).await.is_err(),
"a file is not a folder to create"
);
// Into it and back out, by rename; a full folder is not removed.
let moved = super::path::join(&new, "a2").unwrap();
fs.rename(&a, &moved).await.expect("rename into a folder");
assert!(matches!(fs.stat(&a).await, Err(RemoteError::NotFound(_))));
assert_eq!(fs.stat(&moved).await.expect("moved").size, size_a);
let full = fs.remove_dir(&new).await;
assert!(
full.is_err(),
"a folder with a file in it must not be removed"
);
let removes = !matches!(full, Err(RemoteError::Unsupported(_)));
assert_eq!(fs.stat(&moved).await.expect("still there").size, size_a);
fs.rename(&moved, &a).await.expect("rename back");
if removes {
fs.remove_dir(&new).await.expect("remove the empty folder");
assert!(matches!(fs.stat(&new).await, Err(RemoteError::NotFound(_))));
} else {
assert!(fs.stat(&new).await.expect("left in place").is_dir);
}
// Into a folder that is not there: refused, the file untouched.
let nowhere = format!("{}/x", at("ps5u-missing"));
assert!(fs.rename(&a, &nowhere).await.is_err());
assert_eq!(fs.stat(&a).await.expect("a untouched").size, size_a);
assert!(fs.remove_dir(&at("ps5u-missing")).await.is_err());
// Folders rename too.
let (d1, d2) = (at("ps5u-d1"), at("ps5u-d2"));
fs.create_dir(&d1).await.expect("create d1");
fs.rename(&d1, &d2).await.expect("rename a folder");
assert!(fs.stat(&d2).await.expect("d2").is_dir);
assert!(matches!(fs.stat(&d1).await, Err(RemoteError::NotFound(_))));
if removes {
fs.remove_dir(&d2).await.expect("remove d2");
}
}
@@ -324,6 +324,24 @@ impl FtpFs {
})
}
/// SIZE, then MDTM, on one exact path.
async fn probe(&self, path: &str) -> Probe {
let full = under(&self.home, path);
let mut s = self.session.lock().await;
let size = tokio::time::timeout(STEP, s.size(full.as_str())).await;
let size_absent = match size {
Ok(Ok(n)) => return Probe::Present(Some(n as u64)),
Ok(Err(FtpError::UnexpectedResponse(r))) => r.status.code() == 550,
_ => false,
};
match tokio::time::timeout(STEP, s.mdtm(full.as_str())).await {
Ok(Ok(_)) => Probe::Present(None),
Ok(Err(FtpError::UnexpectedResponse(r))) if r.status.code() == 550 => Probe::Absent,
_ if size_absent => Probe::Absent,
_ => Probe::Unknown,
}
}
async fn entries(&self, path: &str) -> Result<Vec<Entry>, RemoteError> {
let dir = under(&self.home, path);
let dir = dir.as_str();
@@ -351,6 +369,57 @@ impl FtpFs {
}
}
/// What SIZE and MDTM say about one exact path.
#[derive(Debug, Clone, Copy, PartialEq)]
pub(crate) enum Probe {
/// A file is there (with its size when SIZE gave one).
Present(Option<u64>),
/// Both answered "no such file" (550).
Absent,
/// The server answers neither command: only the listing can tell.
Unknown,
}
/// (parent folder, name) of a server path without a trailing `/`.
fn split_parent(p: &str) -> (&str, &str) {
let (parent, name) = p.rsplit_once('/').unwrap_or(("", p));
(if parent.is_empty() { "/" } else { parent }, name)
}
/// Whether a rename may go to `to` (`name` in a folder whose listing is `listing`): only when
/// the folder could be listed (a folder that cannot be is no proof the name is free), no entry
/// in it has the name in any case (a case-insensitive server would replace it), and the exact
/// path probes as absent (a listing can hide dot-files).
pub(crate) fn target_free(
to: &str,
name: &str,
listing: Result<Vec<Entry>, RemoteError>,
probe: Probe,
) -> Result<(), RemoteError> {
let entries = match listing {
Ok(v) => v,
Err(RemoteError::NotFound(m)) => {
return Err(RemoteError::NotFound(format!(
"no folder for {to}, so it cannot be checked ({m})"
)))
}
Err(e) => return Err(e),
};
if let Some(e) = entries
.iter()
.find(|e| e.name == name || e.name.to_lowercase() == name.to_lowercase())
{
return Err(RemoteError::Exists(format!(
"{to} is taken (by {})",
e.name
)));
}
match probe {
Probe::Present(_) => Err(RemoteError::Exists(format!("{to} already exists"))),
Probe::Absent | Probe::Unknown => Ok(()),
}
}
/// A transfer left open mid-file: the stream, the file it reads, and how far it has got.
type OpenTransfer = (
suppaftp::tokio::TransferStream<suppaftp::tokio::AsyncRustlsStream>,
@@ -493,13 +562,25 @@ impl RemoteFs for FtpFs {
mtime: None,
});
}
let (parent, name) = trimmed.rsplit_once('/').unwrap_or(("", trimmed));
let parent = if parent.is_empty() { "/" } else { parent };
self.entries(parent)
let (parent, name) = split_parent(trimmed);
if let Some(e) = self
.entries(parent)
.await?
.into_iter()
.find(|e| e.name == name)
.ok_or_else(|| RemoteError::NotFound(format!("{path} not found")))
{
return Ok(e);
}
// Not in the listing is not proof of absence: LIST hides dot-files on many servers.
match self.probe(trimmed).await {
Probe::Present(size) => Ok(Entry {
name: name.to_string(),
is_dir: false,
size: size.unwrap_or(0),
mtime: None,
}),
_ => Err(RemoteError::NotFound(format!("{path} not found"))),
}
}
async fn open(&self, path: &str) -> Result<Arc<dyn RemoteFile>, RemoteError> {
@@ -563,6 +644,70 @@ impl RemoteFs for FtpFs {
out.sort_by(|a, b| a.0.cmp(&b.0));
Ok(out)
}
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
let full = under(&self.home, path);
let made = {
let mut s = self.session.lock().await;
let host = self.host.as_str();
timed(host, "making a folder", async {
s.mkdir(full.as_str()).await.map_err(|e| map_err(host, e))
})
.await
};
match made {
Ok(()) => Ok(()),
// MKD answers 550 (or 521) for a name already taken; whether that is a folder
// (fine) or a file (not) only a look can tell.
Err(e @ (RemoteError::NotFound(_) | RemoteError::Io(_))) => match self.stat(path).await
{
Ok(st) if st.is_dir => Ok(()),
Ok(_) => Err(RemoteError::Exists(format!("{path} is a file"))),
Err(_) => Err(e),
},
Err(e) => Err(e),
}
}
/// RNFR/RNTO replaces an existing target on many servers (vsftpd and ProFTPD among them),
/// so the target is looked up first and a taken name refused. Something that appears at
/// `to` between that look and the RNTO (a few milliseconds) would still be replaced: FTP
/// has no rename that refuses on its own.
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
let trimmed = to.trim_end_matches('/');
let (parent, name) = split_parent(trimmed);
let listing = self.entries(parent).await;
let probe = if listing.is_ok() {
self.probe(trimmed).await
} else {
Probe::Unknown
};
target_free(to, name, listing, probe)?;
let (src, dst) = (under(&self.home, from), under(&self.home, to));
let mut s = self.session.lock().await;
let host = self.host.as_str();
timed(host, "a rename", async {
s.rename(src.as_str(), dst.as_str())
.await
.map_err(|e| map_err(host, e))
})
.await
}
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
if path.trim_matches('/').is_empty() {
return Err(RemoteError::BadPath(
"the login folder cannot be removed".into(),
));
}
let full = under(&self.home, path);
let mut s = self.session.lock().await;
let host = self.host.as_str();
timed(host, "removing a folder", async {
s.rmdir(full.as_str()).await.map_err(|e| map_err(host, e))
})
.await
}
}
#[cfg(test)]
@@ -573,6 +718,52 @@ mod tests {
v.iter().map(|s| s.to_string()).collect()
}
fn ent(name: &str) -> Entry {
Entry {
name: name.into(),
is_dir: false,
size: 1,
mtime: None,
}
}
#[test]
fn a_rename_target_is_free_only_when_the_server_says_so_every_way() {
let listed = || Ok(vec![ent("Game.pkg"), ent("b")]);
// Free: listed folder without it, and SIZE/MDTM say absent (or cannot say).
assert!(target_free("/g/new.pkg", "new.pkg", listed(), Probe::Absent).is_ok());
assert!(target_free("/g/new.pkg", "new.pkg", listed(), Probe::Unknown).is_ok());
// The same name in another case: a case-insensitive server would replace it.
assert!(matches!(
target_free("/g/GAME.PKG", "GAME.PKG", listed(), Probe::Absent),
Err(RemoteError::Exists(_))
));
// Hidden from the listing (a dot-file), but SIZE finds it.
assert!(matches!(
target_free("/g/.x", ".x", listed(), Probe::Present(Some(3))),
Err(RemoteError::Exists(_))
));
// A folder that cannot be listed proves nothing.
assert!(matches!(
target_free(
"/g/new.pkg",
"new.pkg",
Err(RemoteError::NotFound("550".into())),
Probe::Absent
),
Err(RemoteError::NotFound(_))
));
assert!(target_free(
"/g/new.pkg",
"new.pkg",
Err(RemoteError::Io("421".into())),
Probe::Absent
)
.is_err());
assert_eq!(split_parent("/a.pkg"), ("/", "a.pkg"));
assert_eq!(split_parent("/g/h/a.pkg"), ("/g/h", "a.pkg"));
}
#[test]
fn paths_resolve_under_the_login_folder() {
assert_eq!(under("/", "/games/a.pkg"), "/games/a.pkg");
@@ -651,6 +842,23 @@ mod tests {
.await
.unwrap();
crate::remote::contract::check(&fs, "/").await;
// A writable folder `w/` holding files `a` and `b` of different sizes, and a dot-file
// `.hidden` that the server's LIST leaves out.
if fs.stat("/w/a").await.is_ok() {
crate::remote::contract::check_writes(&fs, "/w").await;
assert!(matches!(
fs.rename("/w/a", "/w/B").await,
Err(RemoteError::Exists(_))
));
if fs.stat("/w/.hidden").await.is_ok() {
assert!(matches!(
fs.rename("/w/a", "/w/.hidden").await,
Err(RemoteError::Exists(_))
));
}
assert!(fs.rename("/w/a", "/nope-7f3a/x").await.is_err());
assert_eq!(fs.stat("/w/a").await.unwrap().size, 3);
}
}
/// FTPS with a self-signed certificate (the delfer image with TLS_CERT/TLS_KEY), then
@@ -22,7 +22,7 @@ pub mod store;
pub mod webdav_fs;
#[cfg(test)]
use std::collections::BTreeMap;
use std::collections::{BTreeMap, BTreeSet};
#[cfg(test)]
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
@@ -62,6 +62,12 @@ pub enum RemoteError {
NotFound(String),
#[error("{0}")]
Io(String),
/// A write would replace something already there (a rename's target, say).
#[error("{0}")]
Exists(String),
/// The server (or this protocol) cannot do what was asked.
#[error("{0}")]
Unsupported(String),
/// The server's identity (SFTP host key, FTPS certificate) is not the one the user accepted.
#[error("Sign-in failed: {}", if *changed { "the server's key has changed since you last connected" } else { "unknown server key — accept it to connect" })]
HostKey { fingerprint: String, changed: bool },
@@ -82,6 +88,31 @@ pub trait RemoteFs: Send + Sync {
async fn open(&self, path: &str) -> Result<Arc<dyn RemoteFile>, RemoteError>;
/// Every regular file under `path`, as (path relative to `path`, entry), capped at `limit`.
async fn walk(&self, path: &str, limit: usize) -> Result<Vec<(String, Entry)>, RemoteError>;
// ── Writes: only what the Collection's organizer needs, all of it non-destructive ──
/// Creates one folder (its parent must exist). A folder already there is success; a
/// file by that name is [`RemoteError::Exists`].
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported(format!(
"this server cannot create {path}"
)))
}
/// Renames a file or folder within the server. Never replaces anything: when `to` is
/// already there it fails with [`RemoteError::Exists`] and both stay as they were.
async fn rename(&self, from: &str, _to: &str) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported(format!(
"this server cannot rename {from}"
)))
}
/// Removes an empty folder. A folder with anything in it is refused and left as it is.
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported(format!(
"this server cannot remove {path}"
)))
}
}
#[cfg(test)]
@@ -92,6 +123,23 @@ pub(crate) struct MemFs {
mtimes: Arc<Mutex<BTreeMap<String, i64>>>,
fail_reads: Arc<AtomicUsize>,
fail_lists: AtomicUsize,
/// Folders made explicitly (an empty one has no file to imply it).
dirs: Arc<Mutex<BTreeSet<String>>>,
/// The next writes happen but their reply is lost: the caller sees a dropped connection.
lose_replies: Arc<AtomicUsize>,
/// Writes until the server goes away (0: never): that write happens, its reply is lost,
/// and every call after it fails until [`MemFs::set_down`] brings the server back.
down_after: Arc<AtomicUsize>,
down: Arc<std::sync::atomic::AtomicBool>,
/// Removes no folder, as WebDAV (whose DELETE is recursive) never does.
keep_dirs: Arc<std::sync::atomic::AtomicBool>,
}
#[cfg(test)]
#[allow(deprecated)] // fetch_update is the pre-1.99 spelling of try_update
fn take_one(n: &AtomicUsize) -> bool {
n.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |n| n.checked_sub(1))
.is_ok()
}
#[cfg(test)]
@@ -103,6 +151,11 @@ impl MemFs {
mtimes: Arc::default(),
fail_reads: Arc::new(AtomicUsize::new(0)),
fail_lists: AtomicUsize::new(0),
dirs: Arc::default(),
lose_replies: Arc::default(),
down_after: Arc::default(),
down: Arc::default(),
keep_dirs: Arc::default(),
}
}
@@ -133,9 +186,63 @@ impl MemFs {
mtimes: Arc::clone(&self.mtimes),
fail_reads: Arc::new(AtomicUsize::new(0)),
fail_lists: AtomicUsize::new(0),
dirs: Arc::clone(&self.dirs),
lose_replies: Arc::clone(&self.lose_replies),
down_after: Arc::clone(&self.down_after),
down: Arc::clone(&self.down),
keep_dirs: Arc::clone(&self.keep_dirs),
}
}
/// From now on folders are never removed (`remove_dir` is unsupported), as on WebDAV.
pub fn never_remove_dirs(&self) {
self.keep_dirs.store(true, Ordering::SeqCst);
}
/// The `n`-th write from now takes effect, loses its reply, and takes the server down.
pub fn go_down_after_writes(&self, n: usize) {
self.down_after.store(n, Ordering::SeqCst);
}
/// Takes the server away (every call fails as a dropped connection) or brings it back.
pub fn set_down(&self, down: bool) {
self.down.store(down, Ordering::SeqCst);
}
fn check_up(&self) -> Result<(), RemoteError> {
if self.down.load(Ordering::SeqCst) {
Err(RemoteError::Io("connection reset".into()))
} else {
Ok(())
}
}
/// The next `n` writes take effect, but report a dropped connection.
pub fn lose_next_replies(&self, n: usize) {
self.lose_replies.store(n, Ordering::SeqCst);
}
/// A file's bytes, for tests to check nothing was overwritten.
pub fn bytes(&self, path: &str) -> Option<Vec<u8>> {
self.files.lock().unwrap().get(&normal(path)).cloned()
}
/// Runs a write: refused outright, or done with its reply lost, as the test asked.
fn write<T>(&self, op: impl FnOnce() -> Result<T, RemoteError>) -> Result<T, RemoteError> {
self.check_up()?;
let r = op()?;
if self.down_after.load(Ordering::SeqCst) > 0
&& self.down_after.fetch_sub(1, Ordering::SeqCst) == 1
{
self.set_down(true);
}
self.check_up()?;
if take_one(&self.lose_replies) {
return Err(RemoteError::Io("connection reset".into()));
}
Ok(r)
}
/// The next `n` listings fail, as a dropped session would.
pub fn fail_next_lists(&self, n: usize) {
self.fail_lists.store(n, Ordering::SeqCst);
@@ -188,11 +295,38 @@ impl MemFs {
}
}
}
drop(files);
for d in self.dirs.lock().unwrap().range(prefix.clone()..) {
let Some(rest) = d.strip_prefix(&prefix) else {
break;
};
let name = rest.split('/').next().unwrap_or(rest);
if !name.is_empty() {
out.entry(name.to_string()).or_insert(Entry {
name: name.to_string(),
is_dir: true,
size: 0,
mtime: None,
});
}
}
out.into_values().collect()
}
fn is_dir(&self, dir: &str) -> bool {
dir == "/" || !self.children(dir).is_empty()
let explicit = self.dirs.lock().unwrap().contains(dir);
dir == "/" || explicit || !self.children(dir).is_empty()
}
fn is_file(&self, p: &str) -> bool {
self.files.lock().unwrap().contains_key(p)
}
fn parent_of(p: &str) -> String {
match p.rsplit_once('/') {
Some(("", _)) | None => "/".into(),
Some((d, _)) => d.into(),
}
}
}
@@ -236,6 +370,7 @@ impl RemoteFile for MemFile {
#[async_trait::async_trait]
impl RemoteFs for MemFs {
async fn list(&self, path: &str, cursor: Option<String>) -> Result<Page, RemoteError> {
self.check_up()?;
if self.list_fails() {
return Err(RemoteError::Io("connection reset".into()));
}
@@ -253,6 +388,7 @@ impl RemoteFs for MemFs {
}
async fn stat(&self, path: &str) -> Result<Entry, RemoteError> {
self.check_up()?;
let p = normal(path);
if let Some(b) = self.files.lock().unwrap().get(&p) {
let name = p.rsplit('/').next().unwrap_or("").to_string();
@@ -317,6 +453,98 @@ impl RemoteFs for MemFs {
})
.collect())
}
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
let p = normal(path);
self.write(|| {
if self.is_file(&p) {
return Err(RemoteError::Exists(format!("{p} is a file")));
}
if self.is_dir(&p) {
return Ok(());
}
if !self.is_dir(&Self::parent_of(&p)) {
return Err(RemoteError::NotFound(format!("no folder for {p}")));
}
self.dirs.lock().unwrap().insert(p.clone());
Ok(())
})
}
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
let (from, to) = (normal(from), normal(to));
self.write(|| {
if self.is_file(&to) || self.is_dir(&to) {
return Err(RemoteError::Exists(format!("{to} already exists")));
}
if !self.is_dir(&Self::parent_of(&to)) {
return Err(RemoteError::NotFound(format!("no folder for {to}")));
}
let mut files = self.files.lock().unwrap();
if let Some(b) = files.remove(&from) {
files.insert(to.clone(), b);
drop(files);
let mut m = self.mtimes.lock().unwrap();
if let Some(t) = m.remove(&from) {
m.insert(to.clone(), t);
}
// The folder it left stays, as on a real server.
self.dirs.lock().unwrap().insert(Self::parent_of(&from));
return Ok(());
}
drop(files);
if from == "/" || !self.is_dir(&from) {
return Err(RemoteError::NotFound(format!("{from} not found")));
}
if to.starts_with(&format!("{from}/")) {
return Err(RemoteError::Io(format!("{to} is inside {from}")));
}
let moved = |k: &String| -> Option<String> {
if *k == from {
Some(to.clone())
} else {
k.strip_prefix(&format!("{from}/"))
.map(|rest| format!("{to}/{rest}"))
}
};
let mut files = self.files.lock().unwrap();
let keys: Vec<String> = files.keys().cloned().collect();
for k in keys {
if let Some(n) = moved(&k) {
let b = files.remove(&k).unwrap();
files.insert(n, b);
}
}
let mut dirs = self.dirs.lock().unwrap();
let keys: Vec<String> = dirs.iter().cloned().collect();
for k in keys {
if let Some(n) = moved(&k) {
dirs.remove(&k);
dirs.insert(n);
}
}
dirs.insert(to.clone());
dirs.insert(Self::parent_of(&from));
Ok(())
})
}
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
let p = normal(path);
if self.keep_dirs.load(Ordering::SeqCst) {
return Err(RemoteError::Unsupported(format!("{p} left in place")));
}
self.write(|| {
if p == "/" || !self.is_dir(&p) {
return Err(RemoteError::NotFound(format!("{p} is not a folder")));
}
if !self.children(&p).is_empty() {
return Err(RemoteError::Io(format!("{p} is not empty")));
}
self.dirs.lock().unwrap().remove(&p);
Ok(())
})
}
}
#[cfg(test)]
@@ -372,8 +600,14 @@ mod tests {
#[tokio::test]
async fn memfs_meets_the_contract() {
let fs = MemFs::new(&[("/g/a.bin", b"0123456789abcdefXYZ"), ("/g/sub/b", b"x")]);
let fs = MemFs::new(&[
("/g/a.bin", b"0123456789abcdefXYZ"),
("/g/sub/b", b"x"),
("/w/a", b"aaa"),
("/w/b", b"bbbbb"),
]);
crate::remote::contract::check(&fs, "/g").await;
crate::remote::contract::check_writes(&fs, "/w").await;
}
#[tokio::test]
@@ -26,13 +26,17 @@ pub fn parse(s: &str) -> Result<RemotePath, RemoteError> {
}
let mut segments = Vec::new();
for raw in raw_path.split('/') {
// A backslash travels encoded (`%5C`, see [`encode_segment`]); a bare one is refused.
if raw.contains('\\') {
return Err(RemoteError::BadPath(format!("bad path segment {raw:?}")));
}
let seg = percent_decode(raw)?;
match seg.as_str() {
"" | "." => continue,
".." => return Err(RemoteError::BadPath(format!("{s} leaves its folder"))),
_ => {}
}
check_segment(&seg)?;
check_name(&seg)?;
segments.push(seg);
}
Ok(RemotePath {
@@ -48,12 +52,35 @@ pub fn format(id: &str, path: &str) -> String {
format!("{SCHEME}{id}/{}", path.trim_start_matches('/'))
}
/// One file or folder name as it goes into a `remote://` path: `%` and `\` encoded, so that
/// [`parse`] gives back exactly this name ("a%20b.pkg" is not "a b.pkg", and a backslash in a
/// name on a Unix server is a name character, not a separator).
pub fn encode_segment(name: &str) -> String {
let mut out = String::with_capacity(name.len());
for c in name.chars() {
match c {
'%' => out.push_str("%25"),
'\\' => out.push_str("%5C"),
c => out.push(c),
}
}
out
}
/// A `/`-separated relative path of `remote://` segments back to the names they stand for.
pub fn decode_rel(rel: &str) -> String {
rel.split('/')
.map(|seg| percent_decode(seg).unwrap_or_else(|_| seg.to_string()))
.collect::<Vec<_>>()
.join("/")
}
/// `base` + one entry name, for walking a tree. The name must be a single plain segment.
pub fn join(base: &str, name: &str) -> Result<String, RemoteError> {
if name.is_empty() || name == "." || name == ".." {
return Err(RemoteError::BadPath(format!("bad name {name:?}")));
}
check_segment(name)?;
check_name(name)?;
let base = base.trim_end_matches('/');
Ok(format!("{base}/{name}"))
}
@@ -66,8 +93,10 @@ fn valid_id(id: &str) -> bool {
.all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_')
}
fn check_segment(seg: &str) -> Result<(), RemoteError> {
if seg.contains(['/', '\\', '\0']) {
/// A name: no `/` or NUL, and no `..` hidden behind a backslash (which a Windows server would
/// read as a separator).
fn check_name(seg: &str) -> Result<(), RemoteError> {
if seg.contains(['/', '\0']) || seg.split('\\').any(|p| p == "..") {
return Err(RemoteError::BadPath(format!("bad path segment {seg:?}")));
}
Ok(())
@@ -124,10 +153,29 @@ mod tests {
assert!(join("/games", "../x").is_err());
assert!(join("/games", "a/b").is_err());
assert!(join("/games", "..").is_err());
assert!(join("/games", "a\\..\\x").is_err());
assert!(parse("remote://nas-1/a%5C..%5Cx").is_err());
assert_eq!(join("/games", "a.pkg").unwrap(), "/games/a.pkg");
assert_eq!(join("/", "a.pkg").unwrap(), "/a.pkg");
}
#[test]
fn every_name_comes_back_as_itself() {
for name in [
"a%20b.pkg",
"a b.pkg",
"100% Orange.pkg",
"a\\b.pkg",
"%25",
"x%",
] {
let s = format!("remote://nas-1/games/{}", encode_segment(name));
assert_eq!(parse(&s).unwrap().path, format!("/games/{name}"), "{name}");
assert_eq!(decode_rel(&encode_segment(name)), name);
}
assert_eq!(join("/g", "a\\b.pkg").unwrap(), "/g/a\\b.pkg");
}
#[test]
fn formats_round_trip() {
assert_eq!(
@@ -253,11 +253,69 @@ pub(crate) fn is_transient(e: &RemoteError) -> bool {
matches!(e, RemoteError::Io(_) | RemoteError::Unreachable(_))
}
/// `m` with `secret` replaced by "***", as typed and as it appears percent-encoded in a URL.
/// A secret of four characters or more goes wherever it appears. A shorter one only where it
/// stands as a whole token (`user:pw@`, `password=pw`, quoted, or right after a `%XX` escape
/// such as `password%3Dpw`): replacing every "t" of a one-letter password turned "Connection"
/// into "Connec***ion".
pub(crate) fn redact(m: &str, secret: &str) -> String {
if secret.is_empty() {
return m.to_string();
}
let encoded = url_encode(secret);
let once = redact_form(m, secret);
if encoded == secret {
once
} else {
redact_form(&once, &encoded)
}
}
fn url_encode(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for b in s.bytes() {
if b.is_ascii_alphanumeric() || b"-._~".contains(&b) {
out.push(b as char);
} else {
out.push_str(&format!("%{b:02X}"));
}
}
out
}
fn redact_form(m: &str, secret: &str) -> String {
if secret.chars().count() >= 4 {
return m.replace(secret, "***");
}
let word = |c: Option<char>| c.is_some_and(|c| c.is_alphanumeric() || c == '_');
// `%3D` right before: the escape ends there, so the secret starts a token.
let after_escape = |i: usize| {
let b = m.as_bytes();
i >= 3 && b[i - 3] == b'%' && b[i - 2].is_ascii_hexdigit() && b[i - 1].is_ascii_hexdigit()
};
let mut out = String::with_capacity(m.len());
let mut last = 0;
for (i, _) in m.match_indices(secret) {
if i < last {
continue;
}
let before = m[..i].chars().next_back();
let after = m[i + secret.len()..].chars().next();
if (!word(before) || after_escape(i)) && !word(after) {
out.push_str(&m[last..i]);
out.push_str("***");
last = i + secret.len();
}
}
out.push_str(&m[last..]);
out
}
/// Replace any secret the server echoed back in an error with "***".
pub fn scrub(e: RemoteError, secret: &Secret) -> RemoteError {
let clean = |mut m: String| {
for s in secret.sensitive() {
m = m.replace(s, "***");
m = redact(&m, s);
}
m
};
@@ -268,6 +326,8 @@ pub fn scrub(e: RemoteError, secret: &Secret) -> RemoteError {
RemoteError::Unreachable(m) => RemoteError::Unreachable(clean(m)),
RemoteError::NotFound(m) => RemoteError::NotFound(clean(m)),
RemoteError::Io(m) => RemoteError::Io(clean(m)),
RemoteError::Exists(m) => RemoteError::Exists(clean(m)),
RemoteError::Unsupported(m) => RemoteError::Unsupported(clean(m)),
e @ RemoteError::HostKey { .. } => e,
}
}
@@ -337,6 +397,15 @@ pub(crate) mod testing {
) -> Result<Vec<(String, crate::remote::Entry)>, RemoteError> {
self.0.walk(p, l).await
}
async fn create_dir(&self, p: &str) -> Result<(), RemoteError> {
self.0.create_dir(p).await
}
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
self.0.rename(from, to).await
}
async fn remove_dir(&self, p: &str) -> Result<(), RemoteError> {
self.0.remove_dir(p).await
}
}
/// A Remote over a fresh temp store and `fs`.
@@ -364,6 +433,35 @@ mod tests {
use crate::remote::store::conn;
use crate::remote::MemFs;
#[test]
fn short_secrets_are_redacted_as_whole_tokens_only() {
let pw = |p: &str| Secret::Password { password: p.into() };
let e = scrub(
RemoteError::Io("Connection error: ftp://t:t@nas/ password=t 't'".into()),
&pw("t"),
);
assert_eq!(
e.to_string(),
"Connection error: ftp://***:***@nas/ password=*** '***'"
);
let e = scrub(
RemoteError::Auth("login hunter2x refused, bad pw hunter2x".into()),
&pw("hunter2x"),
);
assert_eq!(
e.to_string(),
"Sign-in failed: login *** refused, bad pw ***"
);
assert_eq!(redact("abcabc", "abc"), "abcabc");
assert_eq!(redact("x abc y", "abc"), "x *** y");
assert_eq!(redact("nothing here", ""), "nothing here");
// URL-encoded: after an escape, and the secret's own encoded form.
assert_eq!(redact("?password%3Dt&x=1", "t"), "?password%3D***&x=1");
assert_eq!(redact("ftp://me:a%40b@nas", "a@b"), "ftp://me:***@nas");
assert_eq!(redact("user=me&pass=p%26", "p&"), "user=me&pass=***");
assert_eq!(redact("Connection to host", "t"), "Connection to host");
}
#[tokio::test]
async fn reuses_a_session_and_drops_it_on_invalidate() {
let r = remote_with(MemFs::new(&[("/a", b"x")]), None);
@@ -337,6 +337,75 @@ impl RemoteFs for SftpFs {
out.sort_by(|a, b| a.0.cmp(&b.0));
Ok(out)
}
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
let full = under(&self.home, path);
let host = self.host.as_str();
let made = timed(host, "making a folder", async {
self.sftp
.create_dir(full.clone())
.await
.map_err(|e| sftp_err(host, e))
})
.await;
match made {
Ok(()) => Ok(()),
// SFTP v3 answers a taken name with a bare FAILURE: look to see what is there.
Err(e) => match self.stat(path).await {
Ok(st) if st.is_dir => Ok(()),
Ok(_) => Err(RemoteError::Exists(format!("{path} is a file"))),
Err(_) => Err(e),
},
}
}
/// SSH_FXP_RENAME (v3), which refuses an existing target (OpenSSH links the new name, so
/// the refusal is atomic there). The posix-rename extension, which replaces, is never
/// used. The target is also looked up first, for servers that bend the v3 rule.
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
match self.stat(to).await {
Ok(_) => return Err(RemoteError::Exists(format!("{to} already exists"))),
Err(RemoteError::NotFound(_)) => {}
Err(e) => return Err(e),
}
let (src, dst) = (under(&self.home, from), under(&self.home, to));
let host = self.host.as_str();
let done = timed(host, "a rename", async {
self.sftp
.rename(src.clone(), dst.clone())
.await
.map_err(|e| sftp_err(host, e))
})
.await;
match done {
// Something took the name meanwhile, and the file is still where it was.
Err(e)
if !matches!(e, RemoteError::NotFound(_))
&& self.stat(to).await.is_ok()
&& self.stat(from).await.is_ok() =>
{
Err(RemoteError::Exists(format!("{to} already exists ({e})")))
}
other => other,
}
}
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
if path.trim_matches('/').is_empty() {
return Err(RemoteError::BadPath(
"the login folder cannot be removed".into(),
));
}
let full = under(&self.home, path);
let host = self.host.as_str();
timed(host, "removing a folder", async {
self.sftp
.remove_dir(full.clone())
.await
.map_err(|e| sftp_err(host, e))
})
.await
}
}
#[cfg(test)]
@@ -372,6 +441,10 @@ mod tests {
conn.host_key = Some(fingerprint);
let fs = SftpFs::connect(&conn, &secret).await.unwrap();
crate::remote::contract::check(&fs, "/data").await;
// A folder `data/w/` holding files `a` and `b` of different sizes, writable by t.
if fs.stat("/data/w/a").await.is_ok() {
crate::remote::contract::check_writes(&fs, "/data/w").await;
}
let wrong = Secret::Password {
password: "nope".into(),
};
@@ -55,6 +55,7 @@ fn map_err(host: &str, e: smb2::Error) -> RemoteError {
match e.kind() {
K::AuthRequired | K::SigningRequired | K::AccessDenied => RemoteError::Auth(msg),
K::NotFound | K::NotADirectory => RemoteError::NotFound(msg),
K::AlreadyExists => RemoteError::Exists(msg),
K::ConnectionLost | K::TimedOut => RemoteError::Unreachable(format!("{host}: {msg}")),
_ => RemoteError::Io(msg),
}
@@ -245,6 +246,46 @@ impl RemoteFs for SmbFs {
out.sort_by(|a, b| a.0.cmp(&b.0));
Ok(out)
}
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
let made = {
let mut s = self.session.lock().await;
let (client, tree) = &mut *s;
client.create_directory(tree, &share_path(path)).await
};
match made.map_err(|e| map_err(&self.host, e)) {
Err(RemoteError::Exists(m)) => match self.stat(path).await {
Ok(e) if e.is_dir => Ok(()),
_ => Err(RemoteError::Exists(m)),
},
other => other,
}
}
/// SMB2 SET_INFO FileRenameInformation with ReplaceIfExists = false (the crate's only
/// form): the server itself refuses a taken name with OBJECT_NAME_COLLISION.
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
let mut s = self.session.lock().await;
let (client, tree) = &mut *s;
client
.rename(tree, &share_path(from), &share_path(to))
.await
.map_err(|e| map_err(&self.host, e))
}
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
if share_path(path).is_empty() {
return Err(RemoteError::BadPath(
"the share itself cannot be removed".into(),
));
}
let mut s = self.session.lock().await;
let (client, tree) = &mut *s;
client
.delete_directory(tree, &share_path(path))
.await
.map_err(|e| map_err(&self.host, e))
}
}
#[cfg(test)]
@@ -275,8 +316,18 @@ mod tests {
let mut conn = crate::remote::store::conn("t", crate::remote::store::Protocol::Smb);
conn.host = host.into();
conn.port = port.parse().unwrap();
conn.user = String::new();
let fs = SmbFs::connect(&conn, &Secret::None).await.unwrap();
conn.user = std::env::var("PS5UPLOAD_SAMBA_USER").unwrap_or_default();
let secret = match std::env::var("PS5UPLOAD_SAMBA_PASS") {
Ok(password) => Secret::Password { password },
Err(_) => Secret::None,
};
let fs = SmbFs::connect(&conn, &secret).await.unwrap();
crate::remote::contract::check(&fs, "/").await;
// A writable share with files `w/a` and `w/b` of different sizes. The share's folder must
// keep extended attributes: on a Docker Desktop bind mount Samba answers every rename
// with STATUS_NOT_SUPPORTED, so seed the files inside the container.
if fs.stat("/w/a").await.is_ok() {
crate::remote::contract::check_writes(&fs, "/w").await;
}
}
}
@@ -72,9 +72,15 @@ impl RemoteSourceFs {
}
}
/// A `PathBuf` holding a server path, back to the `/`-separated string the server takes.
/// A `PathBuf` holding a server path, back to the `/`-separated string the server takes. Only
/// on Windows is a `\\` a `PathBuf` separator; elsewhere it is part of a name and stays.
fn server_path(p: &Path) -> String {
p.to_string_lossy().replace('\\', "/")
let s = p.to_string_lossy();
if cfg!(windows) {
s.replace('\\', "/")
} else {
s.into_owned()
}
}
fn io_err(e: RemoteError) -> std::io::Error {
@@ -171,25 +177,40 @@ impl SourceFs for RemoteSourceFs {
fn read_dir(&self, p: &Path) -> std::io::Result<Vec<(PathBuf, bool)>> {
let dir = server_path(p);
self.entries(&dir)?
.into_iter()
.map(|e| {
let child = super::path::join(&dir, &e.name).map_err(|err| {
std::io::Error::new(std::io::ErrorKind::InvalidData, err.to_string())
})?;
Ok((PathBuf::from(child), e.is_dir))
})
.collect()
}
}
impl RemoteSourceFs {
/// Every entry of the server folder `dir` (a server path), names exactly as the server
/// gave them. Remembered for the stats that follow.
pub fn entries(&self, dir: &str) -> std::io::Result<Vec<super::Entry>> {
let mut out = Vec::new();
let mut cursor = None;
loop {
let page = self
.handle
.block_on(self.pool.with_fs(&self.store, &self.id, |fs| {
let (dir, cursor) = (dir.clone(), cursor.clone());
let (dir, cursor) = (dir.to_string(), cursor.clone());
async move { fs.list(&dir, cursor).await }
}))
.map_err(io_err)?;
for e in page.entries {
let child = super::path::join(&dir, &e.name).map_err(|err| {
std::io::Error::new(std::io::ErrorKind::InvalidData, err.to_string())
})?;
self.known
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(child.clone(), (e.size, e.is_dir, e.mtime));
out.push((PathBuf::from(child), e.is_dir));
if let Ok(child) = super::path::join(dir, &e.name) {
self.known
.lock()
.unwrap_or_else(|e| e.into_inner())
.insert(child, (e.size, e.is_dir, e.mtime));
}
out.push(e);
}
match page.next_cursor {
Some(c) => cursor = Some(c),
@@ -1,10 +1,11 @@
//! WebDAV (RFC 4918) over plain HTTP or HTTPS, read-only: PROPFIND for listings and stat,
//! ranged GET for reads. Basic auth only, which is what NAS boxes, Nextcloud, Alist and
//! WebDAV (RFC 4918) over plain HTTP or HTTPS: PROPFIND for listings and stat, ranged GET
//! for reads, and for the organizer's few writes MKCOL, MOVE (`Overwrite: F`) and DELETE of an
//! empty collection. Basic auth only, which is what NAS boxes, Nextcloud, Alist and
//! rclone serve; Digest-only servers answer 401 and the hint says so.
//!
//! Paths are absolute on the server (`/remote.php/dav/files/me/games`): the DAV root goes
//! in the connection's start folder, and the browser walks down from there. Nothing here
//! writes.
//! writes file contents.
use std::sync::Arc;
use std::time::Duration;
@@ -21,6 +22,8 @@ const PROPFIND_MAX: u64 = 64 * 1024 * 1024;
pub struct WebdavFs {
agent: ureq::Agent,
/// For PROPFIND and the writes: follows no redirect itself (see [`run_dav`]).
dav: ureq::Agent,
/// `https://nas:5006`, no trailing slash.
base: String,
host: String,
@@ -52,20 +55,24 @@ impl WebdavFs {
base64::engine::general_purpose::STANDARD.encode(format!("{user}:"))
)),
};
let config = ureq::Agent::config_builder()
.tls_config(
ureq::tls::TlsConfig::builder()
.disable_verification(conn.insecure_tls)
.build(),
)
.http_status_as_error(false)
// PROPFIND is not in ureq's method list.
.allow_non_standard_methods(true)
.timeout_global(Some(REQUEST_TIMEOUT))
.max_idle_connections_per_host(8)
.build();
let config = |redirects: u32| {
ureq::Agent::config_builder()
.tls_config(
ureq::tls::TlsConfig::builder()
.disable_verification(conn.insecure_tls)
.build(),
)
.http_status_as_error(false)
// PROPFIND is not in ureq's method list.
.allow_non_standard_methods(true)
.timeout_global(Some(REQUEST_TIMEOUT))
.max_idle_connections_per_host(8)
.max_redirects(redirects)
.build()
};
let fs = Self {
agent: ureq::Agent::new_with_config(config),
agent: ureq::Agent::new_with_config(config(10)),
dav: ureq::Agent::new_with_config(config(0)),
base: format!("{scheme}://{hostport}"),
host: hostport,
auth,
@@ -86,24 +93,25 @@ impl WebdavFs {
/// PROPFIND `path` with `depth` 0 or 1, parsed.
async fn propfind(&self, path: &str, depth: u8) -> Result<Vec<DavItem>, RemoteError> {
let agent = self.agent.clone();
let agent = self.dav.clone();
let base = self.base.clone();
let url = self.url(path);
let auth = self.auth.clone();
let host = self.host.clone();
tokio::task::spawn_blocking(move || {
let mut req = ureq::http::Request::builder()
.method("PROPFIND")
.uri(&url)
.header("Depth", depth.to_string())
.header("Content-Type", "application/xml; charset=utf-8")
.header("User-Agent", "ps5upload");
if let Some(a) = &auth {
req = req.header("Authorization", a);
}
let req = req
.body(PROPFIND_BODY)
.map_err(|e| RemoteError::Io(format!("{host}: {e}")))?;
let mut resp = agent.run(req).map_err(|e| map_err(&host, e))?;
let mut resp = run_dav(&agent, &base, &url, &host, |url| {
let mut req = ureq::http::Request::builder()
.method("PROPFIND")
.uri(url)
.header("Depth", depth.to_string())
.header("Content-Type", "application/xml; charset=utf-8")
.header("User-Agent", "ps5upload");
if let Some(a) = &auth {
req = req.header("Authorization", a);
}
req.body(PROPFIND_BODY)
.map_err(|e| RemoteError::Io(format!("{host}: {e}")))
})?;
let status = resp.status().as_u16();
match status {
207 | 200 => {}
@@ -128,6 +136,42 @@ impl WebdavFs {
.map_err(|e| RemoteError::Io(format!("{e}")))?
}
/// One bodiless request; the status, with sign-in failures already turned into errors.
async fn send(
&self,
method: &'static str,
path: &str,
headers: Vec<(&'static str, String)>,
) -> Result<u16, RemoteError> {
let agent = self.dav.clone();
let base = self.base.clone();
let url = self.url(path);
let auth = self.auth.clone();
let host = self.host.clone();
tokio::task::spawn_blocking(move || {
let resp = run_dav(&agent, &base, &url, &host, |url| {
let mut req = ureq::http::Request::builder()
.method(method)
.uri(url)
.header("User-Agent", "ps5upload");
for (k, v) in &headers {
req = req.header(*k, v);
}
if let Some(a) = &auth {
req = req.header("Authorization", a);
}
req.body(())
.map_err(|e| RemoteError::Io(format!("{host}: {e}")))
})?;
match resp.status().as_u16() {
s @ (401 | 403) => Err(RemoteError::Auth(auth_reason(s))),
s => Ok(s),
}
})
.await
.map_err(|e| RemoteError::Io(format!("{e}")))?
}
/// The entries directly under `path`, excluding `path` itself.
async fn entries(&self, path: &str) -> Result<Vec<Entry>, RemoteError> {
let me = normal(path);
@@ -143,6 +187,37 @@ impl WebdavFs {
}
}
/// Sends `make(url)`, and once more to where a redirect points when that is on the same
/// server. ureq's own redirects drop the Authorization header, and Apache (behind many NAS
/// WebDAV shares) redirects a folder's URL without its trailing slash to the one with it.
fn run_dav<B: ureq::AsSendBody>(
agent: &ureq::Agent,
base: &str,
url: &str,
host: &str,
make: impl Fn(&str) -> Result<ureq::http::Request<B>, RemoteError>,
) -> Result<ureq::http::Response<ureq::Body>, RemoteError> {
let resp = agent.run(make(url)?).map_err(|e| map_err(host, e))?;
if !matches!(resp.status().as_u16(), 301 | 302 | 307 | 308) {
return Ok(resp);
}
let to = resp
.headers()
.get("Location")
.and_then(|v| v.to_str().ok())
.unwrap_or("");
let next = if to.starts_with('/') {
format!("{base}{to}")
} else if to.starts_with(&format!("{base}/")) {
to.to_string()
} else {
return Err(RemoteError::Io(format!(
"{host} redirected to another server ({to})"
)));
};
agent.run(make(&next)?).map_err(|e| map_err(host, e))
}
const PROPFIND_BODY: &str = r#"<?xml version="1.0" encoding="utf-8"?>
<D:propfind xmlns:D="DAV:"><D:prop><D:resourcetype/><D:getcontentlength/><D:getlastmodified/></D:prop></D:propfind>"#;
@@ -524,6 +599,48 @@ impl RemoteFs for WebdavFs {
}
Ok(out)
}
async fn create_dir(&self, path: &str) -> Result<(), RemoteError> {
match self.send("MKCOL", path, Vec::new()).await? {
200 | 201 | 204 => Ok(()),
// 405: something is there already. A folder is fine; a file is not.
405 => match self.stat(path).await {
Ok(e) if e.is_dir => Ok(()),
Ok(_) => Err(RemoteError::Exists(format!("{path} is a file"))),
Err(e) => Err(e),
},
409 => Err(RemoteError::NotFound(format!("no folder for {path}"))),
s => Err(RemoteError::Io(format!(
"{}: MKCOL {path} returned HTTP {s}",
self.host
))),
}
}
/// MOVE with `Overwrite: F`: the server answers 412 rather than replace the target.
async fn rename(&self, from: &str, to: &str) -> Result<(), RemoteError> {
let dest = self.url(&normal(to));
let headers = vec![("Destination", dest), ("Overwrite", "F".to_string())];
match self.send("MOVE", &normal(from), headers).await? {
200 | 201 | 204 => Ok(()),
412 => Err(RemoteError::Exists(format!("{to} already exists"))),
404 => Err(RemoteError::NotFound(format!("{from} not found"))),
409 => Err(RemoteError::NotFound(format!("no folder for {to}"))),
s => Err(RemoteError::Io(format!(
"{}: MOVE {from} returned HTTP {s}",
self.host
))),
}
}
/// Never: DELETE on a collection removes it with everything in it, so even after a
/// listing showed it empty, a file landing in the moment before the DELETE would go too.
/// An emptied folder is left in place.
async fn remove_dir(&self, path: &str) -> Result<(), RemoteError> {
Err(RemoteError::Unsupported(format!(
"{path} left in place: WebDAV can only remove a folder with everything in it"
)))
}
}
#[cfg(test)]
@@ -716,4 +833,30 @@ mod tests {
let walked = fs.walk("/d", 10).await.unwrap();
assert!(walked.iter().any(|(p, _)| p == "a b.bin"));
}
/// A real server: `docker run -d -p 8088:80 -e AUTH_TYPE=Basic -e USERNAME=t -e PASSWORD=t
/// bytemark/webdav` with files `w/a` and `w/b` of different sizes in its /var/lib/dav/data,
/// then `PS5UPLOAD_WEBDAV_TEST=127.0.0.1:8088`.
#[tokio::test]
#[ignore = "needs a WebDAV server; see the doc comment"]
async fn webdav_meets_the_remote_contract() {
let Ok(addr) = std::env::var("PS5UPLOAD_WEBDAV_TEST") else {
return;
};
let (host, port) = addr.split_once(':').unwrap();
let mut conn = crate::remote::store::conn("t", Protocol::Webdav);
conn.host = host.into();
conn.port = port.parse().unwrap();
conn.user = "t".into();
let fs = WebdavFs::connect(
&conn,
&Secret::Password {
password: "t".into(),
},
)
.await
.unwrap();
crate::remote::contract::check(&fs, "/").await;
crate::remote::contract::check_writes(&fs, "/w").await;
}
}