Skip to content

Commit bde84d7

Browse files
OldesOldes
committed
FEAT: included a module with *-thru functions for accessing files through a local disk cache
resolves: Oldes/Rebol-issues#2554
1 parent fc55543 commit bde84d7

File tree

4 files changed

+174
-0
lines changed

4 files changed

+174
-0
lines changed

‎src/boot/sysobj.reb

+1
Original file line numberDiff line numberDiff line change
@@ -257,6 +257,7 @@ modules: object [
257257
prebol: https://src.rebol.tech/modules/prebol.reb
258258
scheduler: https://src.rebol.tech/modules/scheduler.reb
259259
spotify: https://src.rebol.tech/modules/spotify.reb
260+
thru-cache: https://src.rebol.tech/modules/thru-cache.reb
260261
to-ascii: https://src.rebol.tech/modules/to-ascii.reb
261262
unicode-utils: https://src.rebol.tech/modules/unicode-utils.reb
262263
daytime: https://src.rebol.tech/mezz/prot-daytime.reb

‎src/modules/thru-cache.reb

+125
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
Rebol [
2+
title: "Thru Cache functions"
3+
purpose: "Functions for accessing remote files through a local cache"
4+
name: thru-cache
5+
type: module
6+
options: [delay]
7+
version: 0.1.0
8+
exports: [path-thru read-thru load-thru do-thru clear-thru]
9+
author: @Oldes
10+
file: %thru-cache.reb
11+
home: https://src.rebol.tech/modules/thru-cache.reb
12+
]
13+
14+
path-thru: func [
15+
{Returns the local disk cache path of a remote file}
16+
url [url!] "Remote file address"
17+
return: [file!]
18+
/local so hash file path
19+
][
20+
so: system/options
21+
unless select so 'thru-cache [
22+
put so 'thru-cache join to-real-file any [
23+
get-env "TEMP"
24+
so/data
25+
] %thru-cache/
26+
sys/log/info 'REBOL ["Using thru-cache:" mold so/thru-cache]
27+
]
28+
hash: checksum form url 'MD5
29+
file: head (remove back tail remove remove (form hash))
30+
path: dirize append copy so/thru-cache copy/part file 2
31+
unless exists? path [make-dir/deep path]
32+
append path file
33+
]
34+
35+
read-thru: func [
36+
"Reads a remote file through local disk cache"
37+
url [url!] "Remote file address"
38+
/update "Force a cache update"
39+
/string "Try convert result to string"
40+
/local path data
41+
][
42+
path: path-thru url
43+
either all [not update exists? path] [
44+
data: read/binary path
45+
][
46+
data: read/binary/all url
47+
if all [
48+
block? data
49+
object? data/2
50+
binary? data/3
51+
][
52+
data: any [
53+
attempt [decompress data/3 to word! data/2/Content-Encoding]
54+
data/3
55+
]
56+
]
57+
try [
58+
write/binary path data
59+
log-thru-file path url
60+
]
61+
]
62+
if string [try [data: to string! data]]
63+
data
64+
]
65+
66+
load-thru: func [
67+
"Loads a remote file through local disk cache"
68+
url [url!] "Remote file address"
69+
/update "Force a cache update"
70+
/as {Specify the type of data; use NONE to load as code}
71+
type [word! none!] "E.g. text, markup, jpeg, unbound, etc."
72+
][
73+
load/as read-thru/:update url type
74+
]
75+
76+
do-thru: func [
77+
{Evaluates a remote Rebol script through local disk cache}
78+
url [url!] "Remote file address"
79+
/update "Force a cache update"
80+
][
81+
do read-thru/:update url
82+
]
83+
84+
clear-thru: func [
85+
"Removes local disk cache files"
86+
/only filter [string!] "Delete only files where the filter is found"
87+
/test "Only print files to be deleted"
88+
/local temp dir log
89+
][
90+
unless exists? dir: select system/options 'thru-cache [exit]
91+
log: dir/read-thru.log
92+
if test [
93+
unless exists? log [exit]
94+
filter: any [filter "*"]
95+
foreach [path url] transcode read log [
96+
if all [
97+
exists? dir/:path
98+
find/any url filter
99+
][ print [as-green skip path 3 url] ]
100+
]
101+
exit
102+
]
103+
either only [
104+
unless exists? log [exit]
105+
;; Go through the log file and decide which files should be deleted
106+
temp: transcode read log
107+
delete log
108+
foreach [path url] temp [
109+
if exists? dir/:path [
110+
either find/any url filter [
111+
try [delete dir/:path] ;; delete the local file
112+
][ log-thru-file path url ] ;; write the path to the new log file
113+
]
114+
]
115+
][ ;; Delete everything
116+
delete-dir dir
117+
]
118+
()
119+
]
120+
121+
log-thru-file: func[path url][
122+
write/append system/options/thru-cache/read-thru.log ajoin [
123+
#"%" copy/part tail path -32 SP url LF
124+
]
125+
]

‎src/tests/run-tests.r3

+1
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ dt [ ;- delta time
6868
%units/url-test.r3
6969
%units/vector-test.r3
7070
%units/word-test.r3
71+
%units/thru-cache-test.r3
7172
;- cryptography tests:
7273
%units/aes-test.r3
7374
%units/chacha20-test.r3

‎src/tests/units/thru-cache-test.r3

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
Rebol [
2+
Title: "Rebol3 word test script"
3+
Author: "Oldes, Peter W A Wood"
4+
File: %thru-cache-test.r3
5+
Tabs: 4
6+
Needs: [%../quick-test-module.r3]
7+
]
8+
9+
~~~start-file~~~ "THRU-CACHE"
10+
11+
if module? try [import 'thru-cache][
12+
;@@ https://github.com/Oldes/Rebol-issues/issues/2554
13+
===start-group=== "thru cache functions"
14+
--test-- "read-thru"
15+
url: https://raw.githubusercontent.com/Oldes/Rebol3/master/src/tests/units/files/%C4%8De%C5%A1tina.txt
16+
--assert try [all [
17+
binary? read-thru url
18+
'file = exists? path-thru url
19+
string? str1: read-thru/string url
20+
string? str2: read/string path-thru url
21+
equal? str1 str2
22+
equal? str1 read/string url
23+
]]
24+
25+
--test-- "load-thru"
26+
url: https://raw.githubusercontent.com/Oldes/Rebol3/master/src/tests/units/files/print-args.r3
27+
--assert try [all [
28+
block? blk1: load-thru url
29+
block? blk2: load url
30+
equal? blk1 blk2
31+
]]
32+
33+
--test-- "do-thru"
34+
--assert not error? try [do-thru url] ;; evaluates the previously downloaded and cached script
35+
36+
--test-- "clear-thru"
37+
clear-thru/test ;; just prints all cached files
38+
--assert 'file = exists? path-thru url
39+
clear-thru/only "*.txt" ;; removes all *.txt files from the cache
40+
--assert 'file = exists? path-thru url
41+
clear-thru ;; removes everything
42+
--assert none? exists? path-thru url
43+
44+
===end-group===
45+
]
46+
47+
~~~end-file~~~

0 commit comments

Comments
 (0)