ENG-1490 (#44)

* Updata package * Update package
PreternaturalAI · Jan 13, 2025 · 074e4d0 · 074e4d0
1 parent 9baf469
commit 074e4d0
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 5 deletions.
diff --git a/Sources/HuggingFace/HuggingFace.Hub.Client.swift b/Sources/HuggingFace/HuggingFace.Hub.Client.swift
@@ -10,7 +10,7 @@ import Foundation
 extension HuggingFace.Hub {
     public struct Client {
         var downloadBase: URL
-        var hfToken: String?
+        public var hfToken: String?
         var endpoint: String
         var useBackgroundSession: Bool
 
@@ -204,7 +204,59 @@ public extension HuggingFace.Hub.Client {
             return destination
         }
     }
+
+    /*
+    @discardableResult
+    func download(
+        from repo: Repo,
+        matching globs: [String] = [],
+        session: URLSession,
+    outputHandler: @escaping (Progress) -> Void = { _ in }
+    ) async throws {
+        let filenames = try await getFilenames(from: repo, matching: globs)
+        let progress = Progress(totalUnitCount: Int64(filenames.count))
+        let repoDestination = localRepoLocation(repo)
+
+        for filename in filenames {
+            let fileProgress = Progress(totalUnitCount: 100, parent: progress, pendingUnitCount: 1)
+            let downloader = HubFileDownloader(
+                repo: repo,
+                repoDestination: repoDestination,
+                relativeFilename: filename,
+                hfToken: hfToken,
+                endpoint: endpoint,
+                backgroundSession: useBackgroundSession
+            )
+            try await downloader.download { fractionDownloaded in
+                fileProgress.completedUnitCount = Int64(100 * fractionDownloaded)
+                outputHandler(progress)
+            }
+            fileProgress.completedUnitCount = 100
+        }
+    }
+    */
+    func formRequest(repo: HuggingFace.Hub.Repo, relativeFilename: String, authToken: String?) -> URLRequest {
+        var url: URL {
+            // https://huggingface.co/coreml-projects/Llama-2-7b-chat-coreml/resolve/main/tokenizer.json?download=true
+            var url = URL(string: endpoint)!
+            if repo.type != .models {
+                url = url.appending(component: repo.type.rawValue)
+            }
+            url = url.appending(path: repo.id)
+            url = url.appending(path: "resolve/main") // TODO: revisions
+            url = url.appending(path: relativeFilename)
+            return url
+        }
+
+        var request = URLRequest(url: url)
+        if let authToken = authToken {
+            request.setValue("Bearer \(authToken)", forHTTPHeaderField: "Authorization")
+        }
+
+        return request
+    }
 
+    // main snapshot function
     @discardableResult
     func snapshot(
         from repo: Repo,

diff --git a/Sources/HuggingFace/HuggingFace.Hub.swift b/Sources/HuggingFace/HuggingFace.Hub.swift
@@ -19,15 +19,15 @@ public extension HuggingFace.Hub {
         case httpStatusCode(Int)
     }
 
-    enum RepoType: String {
+    enum RepoType: String, Codable {
         case models
         case datasets
         case spaces
     }
 
-    struct Repo {
-        let id: String
-        let type: RepoType
+    struct Repo: Codable {
+        public let id: String
+        public let type: RepoType
 
         public init(id: String, type: RepoType = .models) {
             self.id = id