Source code

001package net.filebot.format;
002
003import static java.nio.charset.StandardCharsets.*;
004import static java.util.Arrays.*;
005import static java.util.Collections.*;
006import static java.util.stream.Collectors.*;
007import static net.filebot.CachedResource.*;
008import static net.filebot.Settings.*;
009import static net.filebot.media.XattrMetaInfo.*;
010import static net.filebot.similarity.Normalization.*;
011import static net.filebot.util.FileUtilities.*;
012import static net.filebot.util.RegularExpressions.*;
013import static net.filebot.util.ZipUtilities.*;
014
015import java.io.File;
016import java.net.URI;
017import java.nio.ByteBuffer;
018import java.util.AbstractList;
019import java.util.AbstractMap;
020import java.util.LinkedHashMap;
021import java.util.List;
022import java.util.Locale;
023import java.util.Map;
024import java.util.RandomAccess;
025import java.util.Set;
026import java.util.function.Function;
027import java.util.regex.Pattern;
028
029import org.jsoup.Jsoup;
030
031import groovy.json.JsonSlurper;
032import groovy.xml.XmlSlurper;
033
034import net.filebot.Cache;
035import net.filebot.CacheType;
036import net.filebot.InvalidInputException;
037import net.filebot.MemoryCache;
038import net.filebot.Resource;
039import net.filebot.platform.mac.MacAppUtilities;
040
041public abstract class DataResource {
042
043        public abstract Object getResource();
044
045        public abstract boolean isStale();
046
047        public abstract byte[] bytes() throws Exception;
048
049        public Map<Object, Object> csv() throws Exception {
050                Map<Object, Object> value = new LinkedHashMap<Object, Object>();
051
052                // read CSV file
053                List<Pattern> delimiter = asList(TAB, EQUALS, SEMICOLON, PIPE, COLON, COMMA);
054
055                for (String line : lines()) {
056                        if (line.startsWith("#")) {
057                                continue;
058                        }
059
060                        for (Pattern d : delimiter) {
061                                String[] field = d.split(line, 2);
062                                if (field.length >= 2) {
063                                        // add key=value match
064                                        value.put(field[0], field[1]);
065                                        // lock delimiter
066                                        delimiter = singletonList(d);
067                                        break;
068                                }
069                        }
070                }
071
072                return value;
073        }
074
075        public String text() throws Exception {
076                ByteBuffer data = ByteBuffer.wrap(bytes());
077
078                // users passing Microsoft Excel spreadsheet files instead of plain/text CSV files is surprisingly common
079                if (isZipFile(data)) {
080                        throw new InvalidInputException(this + " is a ZIP archive and not a plain/text file");
081                }
082
083                return decodeTextContent(data, true, UTF_8);
084        }
085
086        public List<String> lines() throws Exception {
087                return asList(NEWLINE.split(text()));
088        }
089
090        public Object xml() throws Exception {
091                return new XmlSlurper().parseText(text());
092        }
093
094        public Object json() throws Exception {
095                return new JsonSlurper().parseText(text());
096        }
097
098        public Object html() throws Exception {
099                return Jsoup.parse(text());
100        }
101
102        public static DataResource local(File file) throws Exception {
103                return getDataResource(file, f -> {
104                        if (isMacSandbox()) {
105                                MacAppUtilities.askUnlockFolders(null, singleton(file));
106                        }
107                        return new Memoized(new Local(f));
108                });
109        }
110
111        public static DataResource remote(URI uri) throws Exception {
112                return getDataResource(uri, f -> {
113                        return new Memoized(new Remote(f));
114                });
115        }
116
117        private static <T> DataResource getDataResource(T resource, Function<T, DataResource> type) throws Exception {
118                DataResource data = cache.getIfPresent(resource);
119                if (data == null || data.isStale()) {
120                        data = type.apply(resource);
121                        cache.put(resource, data);
122                }
123                return data;
124        }
125
126        private static final MemoryCache<Object, DataResource> cache = MemoryCache.forMinutes();
127
128        private static class Local extends DataResource {
129
130                private final File file;
131                private final boolean directory;
132                private final long lastModified;
133
134                public Local(File file) {
135                        this.file = file;
136                        this.directory = file.isDirectory();
137                        this.lastModified = file.lastModified();
138                }
139
140                @Override
141                public Object getResource() {
142                        return file;
143                }
144
145                @Override
146                public boolean isStale() {
147                        return this.lastModified != file.lastModified();
148                }
149
150                @Override
151                public byte[] bytes() throws Exception {
152                        if (!file.isFile()) {
153                                throw new InvalidInputException("File not found: " + file);
154                        }
155                        if (file.length() > ONE_GIGABYTE) {
156                                throw new InvalidInputException("File is too large and probably not a plain/text file: " + file);
157                        }
158                        return readFile(file);
159                }
160
161                @Override
162                public Map<Object, Object> csv() throws Exception {
163                        if (directory) {
164                                return getMediaIndex();
165                        }
166                        return super.csv();
167                }
168
169                @Override
170                public List<String> lines() throws Exception {
171                        if (directory) {
172                                return getDirectoryIndex();
173                        }
174                        return super.lines();
175                }
176
177                public List<String> getDirectoryIndex() {
178                        return asList(file.list());
179                }
180
181                public Map<Object, Object> getMediaIndex() {
182                        Map<Object, Object> value = new LinkedHashMap<Object, Object>();
183                        // read xattr file structure
184                        for (File f : listFiles(file, NOT_HIDDEN)) {
185                                Object m = xattr.getMetaInfo(f);
186                                if (m != null) {
187                                        value.put(f, m);
188                                }
189                        }
190                        return value;
191                }
192        }
193
194        private static class Remote extends DataResource {
195
196                private final URI url;
197
198                public Remote(URI url) {
199                        this.url = url;
200                }
201
202                @Override
203                public Object getResource() {
204                        return url;
205                }
206
207                @Override
208                public boolean isStale() {
209                        return false;
210                }
211
212                @Override
213                public byte[] bytes() throws Exception {
214                        return Cache.getConcurrentCache(Cache.URL, CacheType.Monthly).url(url.toURL()).get();
215                }
216        }
217
218        public static class Post extends DataResource {
219
220                private final URI url;
221                private final String postData;
222                private final String contentType;
223                private final Map<String, String> requestHeader;
224
225                public Post(URI url, String postData, String contentType, Map<String, String> requestHeader) {
226                        this.url = url;
227                        this.postData = postData;
228                        this.contentType = contentType;
229                        this.requestHeader = requestHeader;
230                }
231
232                @Override
233                public Object getResource() {
234                        return url;
235                }
236
237                @Override
238                public boolean isStale() {
239                        return false;
240                }
241
242                @Override
243                public byte[] bytes() throws Exception {
244                        return Cache.getConcurrentCache(Cache.URL, CacheType.Monthly).bytes(url + " " + postData, k -> url.toURL()).fetch(post(() -> {
245                                return postData.getBytes(UTF_8);
246                        }, () -> {
247                                return contentType;
248                        }, () -> {
249                                return requestHeader;
250                        })).get();
251                }
252        }
253
254        private static class Memoized extends DataResource {
255
256                private final DataResource resource;
257
258                private final Resource<Map<Object, Object>> csv;
259                private final Resource<List<String>> lines;
260
261                private final Resource<String> text;
262                private final Resource<Object> xml;
263                private final Resource<Object> json;
264                private final Resource<Object> html;
265
266                public Memoized(DataResource resource) {
267                        this.resource = resource;
268
269                        this.csv = Resource.lazy(() -> new LookupMap(resource.csv()));
270                        this.lines = Resource.lazy(() -> new LookupList(resource.lines()));
271
272                        this.text = Resource.lazy(resource::text);
273                        this.xml = Resource.lazy(resource::xml);
274                        this.json = Resource.lazy(resource::json);
275                        this.html = Resource.lazy(resource::html);
276                }
277
278                @Override
279                public Object getResource() {
280                        return resource.getResource();
281                }
282
283                @Override
284                public boolean isStale() {
285                        return resource.isStale();
286                }
287
288                @Override
289                public byte[] bytes() throws Exception {
290                        return resource.bytes();
291                }
292
293                @Override
294                public Map<Object, Object> csv() throws Exception {
295                        return csv.get();
296                }
297
298                @Override
299                public List<String> lines() throws Exception {
300                        return lines.get();
301                }
302
303                @Override
304                public String text() throws Exception {
305                        return text.get();
306                }
307
308                @Override
309                public Object xml() throws Exception {
310                        return xml.get();
311                }
312
313                @Override
314                public Object json() throws Exception {
315                        return json.get();
316                }
317
318                @Override
319                public Object html() throws Exception {
320                        return html.get();
321                }
322        }
323
324        protected static class LookupMap extends AbstractMap<Object, Object> {
325
326                private final Map<Object, Object> values;
327
328                public LookupMap(Map<Object, Object> values) {
329                        this.values = values;
330                }
331
332                private String definingKey(Object key) {
333                        // letters and digits are defining, everything else will be ignored
334                        return normalizePunctuation(key.toString()).toLowerCase(Locale.ROOT);
335                }
336
337                private Map<String, Object> lookup;
338
339                private Map<String, Object> getLookup() {
340                        if (lookup == null) {
341                                lookup = values.entrySet().stream().collect(toMap(e -> definingKey(e.getKey()), e -> e.getValue(), (a, b) -> a, LinkedHashMap::new));
342                        }
343                        return lookup;
344                }
345
346                @Override
347                public Object get(Object key) {
348                        return getLookup().get(definingKey(key));
349                }
350
351                @Override
352                public boolean containsKey(Object key) {
353                        return getLookup().containsKey(definingKey(key));
354                }
355
356                public Set<Object> keySet() {
357                        return unmodifiableSet(getLookup().keySet());
358                }
359
360                @Override
361                public Set<Entry<Object, Object>> entrySet() {
362                        return unmodifiableSet(values.entrySet());
363                }
364        }
365
366        protected static class LookupList extends AbstractList<String> implements RandomAccess {
367
368                private final List<String> values;
369
370                public LookupList(List<String> values) {
371                        this.values = values;
372                }
373
374                @Override
375                public String get(int index) {
376                        return values.get(index);
377                }
378
379                @Override
380                public int size() {
381                        return values.size();
382                }
383
384                private String definingKey(Object key) {
385                        // letters and digits are defining, everything else will be ignored
386                        return normalizePunctuation(key.toString()).toLowerCase(Locale.ROOT);
387                }
388
389                private Set<String> lookup;
390
391                private Set<String> getLookup() {
392                        if (lookup == null) {
393                                lookup = values.stream().map(this::definingKey).collect(toSet());
394                        }
395                        return lookup;
396                }
397
398                public Set<String> keySet() {
399                        return unmodifiableSet(getLookup());
400                }
401
402                @Override
403                public boolean contains(Object object) {
404                        if (object == null) {
405                                return false;
406                        }
407                        return getLookup().contains(definingKey(object));
408                }
409        }
410
411}