Source code

001package net.filebot.postprocess;
002
003import static java.nio.charset.StandardCharsets.*;
004import static java.util.Collections.*;
005import static java.util.stream.Collectors.*;
006import static net.filebot.Logging.*;
007import static net.filebot.MediaTypes.*;
008import static net.filebot.format.ExpressionFormatFunctions.*;
009import static net.filebot.util.FileUtilities.*;
010import static net.filebot.util.RegularExpressions.*;
011
012import java.io.File;
013import java.net.URI;
014import java.net.URL;
015import java.nio.ByteBuffer;
016import java.nio.file.Files;
017import java.nio.file.StandardOpenOption;
018import java.util.ArrayList;
019import java.util.Collection;
020import java.util.Iterator;
021import java.util.LinkedHashMap;
022import java.util.List;
023import java.util.Locale;
024import java.util.Map;
025import java.util.Objects;
026import java.util.Optional;
027import java.util.function.Function;
028import java.util.function.Predicate;
029import java.util.regex.Pattern;
030
031import org.codehaus.groovy.runtime.DefaultGroovyMethods;
032import org.codehaus.groovy.runtime.InvokerHelper;
033import org.codehaus.groovy.runtime.typehandling.DefaultTypeTransformation;
034
035import groovy.json.JsonOutput;
036import groovy.json.JsonSlurper;
037import groovy.lang.Closure;
038import groovy.lang.GroovyObjectSupport;
039import groovy.lang.Script;
040import groovy.xml.XmlSlurper;
041
042import net.filebot.Execute;
043import net.filebot.Language;
044import net.filebot.StandardRenameAction;
045import net.filebot.UserFiles;
046import net.filebot.UserInteraction;
047import net.filebot.WebServices;
048import net.filebot.subtitle.SubtitleFormat;
049import net.filebot.subtitle.SubtitleUtilities;
050import net.filebot.util.Builder;
051import net.filebot.util.ByteBufferInputStream;
052import net.filebot.vfs.MemoryFile;
053import net.filebot.web.SubtitleDescriptor;
054import net.filebot.web.SubtitleLookupService;
055import net.filebot.web.SubtitleProvider;
056import net.filebot.web.WebRequest;
057
058public abstract class ScriptBaseClass extends Script {
059
060        public void alert(Object... messages) throws Exception {
061                String message = message(messages).toString();
062
063                println("ALERT " + message);
064                log.warning(message);
065        }
066
067        public void system(String executable, Object... args) throws Exception {
068                system(EMPTY_MAP, executable, args);
069        }
070
071        public void system(Map env, String executable, Object... args) throws Exception {
072                Collection<Object> values = DefaultGroovyMethods.flatten(args);
073                List<String> arguments = values.stream().filter(Objects::nonNull).map(Objects::toString).collect(toList());
074
075                println("EXECUTE " + executable + " " + arguments);
076                Execute.system(executable, arguments, null, toStringMap(env));
077        }
078
079        public void trash(Object... args) throws Exception {
080                for (File f : asFileList(args)) {
081                        println("TRASH " + f);
082                        UserFiles.trash(f);
083                }
084        }
085
086        public void move(Object... args) throws Exception {
087                Iterator<File> iterator = asFileList(args).iterator();
088                while (iterator.hasNext()) {
089                        File from = iterator.next();
090                        File to = iterator.next();
091
092                        if (StandardRenameAction.MOVE.canRename(from, to)) {
093                                if (from.exists() && to.exists()) {
094                                        trash(to);
095                                }
096                                println("MOVE " + from + " to " + to);
097                                StandardRenameAction.MOVE.rename(from, to);
098                        }
099                }
100        }
101
102        public void reveal(Object... args) throws Exception {
103                for (File f : asFileList(args)) {
104                        println("REVEAL " + f);
105                        UserInteraction.reveal(f);
106                }
107        }
108
109        public File curl(Object url, File file) throws Exception {
110                if (file.exists()) {
111                        println("SKIP " + file);
112                        return null;
113                }
114
115                println("GET " + url + " (" + file + ")");
116                ByteBuffer bytes = WebRequest.fetch(url(url));
117
118                createFolders(file.getParentFile());
119                return writeFile(bytes, file);
120        }
121
122        public Response curl(Object url) throws Exception {
123                return request(url, EMPTY_MAP, null);
124        }
125
126        public Response curl(Map header, Object url) throws Exception {
127                return request(url, header, null);
128        }
129
130        public Response curl(Object url, Map json) throws Exception {
131                return request(url, EMPTY_MAP, Post.of(WebRequest.HTTP_POST, json, EMPTY_MAP));
132        }
133
134        public Response curl(Object url, String text) throws Exception {
135                return request(url, EMPTY_MAP, Post.of(WebRequest.HTTP_POST, text, EMPTY_MAP));
136        }
137
138        public Response submit(Object url, Map form) throws Exception {
139                return request(url, EMPTY_MAP, Post.of(WebRequest.HTTP_POST, form, singletonMap("Content-Type", "application/x-www-form-urlencoded")));
140        }
141
142        public Response curl(Map header, Object url, Object data) throws Exception {
143                return request(url, header, Post.of(WebRequest.HTTP_POST, data, header));
144        }
145
146        public Response http(String method, Object url, Object data) throws Exception {
147                return request(url, EMPTY_MAP, Post.of(method, data, EMPTY_MAP));
148        }
149
150        public Response http(Map header, String method, Object url, Object data) throws Exception {
151                return request(url, header, Post.of(method, data, header));
152        }
153
154        private Response request(Object url, Map header, Post data) throws Exception {
155                URL resource = url(url);
156
157                Map<String, String> responseHeader = new LinkedHashMap<String, String>();
158
159                // send HTTP GET request
160                if (data == null) {
161                        println("GET " + resource);
162
163                        ByteBuffer response = WebRequest.fetch(resource, 0, null, toStringMap(header), responseHeader::put);
164                        return new Response(response, responseHeader);
165                }
166
167                // send HTTP POST request
168                println(data.getRequestMethod() + " " + resource + " " + data.toString());
169
170                ByteBuffer response = WebRequest.post(data.getRequestMethod(), resource, data.toByteArray(), data.getContentType(), toStringMap(header), responseHeader::put);
171                return new Response(response, responseHeader);
172        }
173
174        public static class Response extends GroovyObjectSupport implements Iterable<Object> {
175
176                private final ByteBuffer content;
177                private final Map<String, String> header;
178
179                private Object object;
180
181                public Response(ByteBuffer content, Map<String, String> header) {
182                        this.content = content;
183                        this.header = header;
184                }
185
186                public ByteBuffer content() {
187                        return content.duplicate();
188                }
189
190                public Map<String, String> header() {
191                        return header;
192                }
193
194                public String header(String name) {
195                        // make response headers case-insensitive
196                        return header().getOrDefault(name.toLowerCase(Locale.ROOT), "");
197                }
198
199                public synchronized Object object() {
200                        if (object == null) {
201                                object = ContentType.forContentType(header("content-type")).parse(content());
202                        }
203                        return object;
204                }
205
206                @Override
207                public Object invokeMethod(String name, Object args) {
208                        return InvokerHelper.invokeMethod(object(), name, args);
209                }
210
211                @Override
212                public Object getProperty(String property) {
213                        return InvokerHelper.getProperty(object(), property);
214                }
215
216                @Override
217                public Iterator<Object> iterator() {
218                        return DefaultGroovyMethods.iterator(object());
219                }
220
221                public boolean asBoolean() {
222                        return DefaultTypeTransformation.castToBoolean(object());
223                }
224
225                @Override
226                public String toString() {
227                        return UTF_8.decode(content()).toString();
228                }
229        }
230
231        private static class Post {
232
233                private final String requestMethod;
234                private final Object content;
235                private final String contentType;
236
237                private Post(String requestMethod, Object content, String contentType) {
238                        this.requestMethod = requestMethod;
239                        this.content = content;
240                        this.contentType = contentType;
241                }
242
243                public String getRequestMethod() {
244                        return requestMethod;
245                }
246
247                public String getContentType() {
248                        return contentType;
249                }
250
251                public byte[] toByteArray() {
252                        // application/octet-stream
253                        if (content instanceof byte[]) {
254                                return (byte[]) content;
255                        }
256
257                        // application/json or application/x-www-form-urlencoded or text/plain
258                        return content.toString().getBytes(UTF_8);
259                }
260
261                @Override
262                public String toString() {
263                        // application/octet-stream
264                        if (content instanceof byte[]) {
265                                byte[] bytes = (byte[]) content;
266                                return "[" + bytes.length + " bytes]";
267                        }
268
269                        // application/json or application/x-www-form-urlencoded or text/plain
270                        return content.toString();
271                }
272
273                public static Post of(String method, Object data, Map header) {
274                        if (data instanceof Map && "application/x-www-form-urlencoded".equals(header.get("Content-Type"))) {
275                                return new Post(method, WebRequest.encodeParameters((Map) data), "application/x-www-form-urlencoded");
276                        }
277                        if (data instanceof CharSequence) {
278                                return new Post(method, data.toString(), "text/plain");
279                        }
280                        if (data instanceof byte[]) {
281                                return new Post(method, (byte[]) data, "application/octet-stream");
282                        }
283                        if (data == null) {
284                                return new Post(method, new byte[0], "application/octet-stream");
285                        }
286                        // post as application/json by default
287                        return new Post(method, JsonOutput.toJson(data), "application/json");
288                }
289
290        }
291
292        public static enum ContentType {
293
294                JSON, XML, TEXT, BYTES;
295
296                public Object parse(ByteBuffer bytes) {
297                        try {
298                                switch (this) {
299                                        case JSON:
300                                                return new JsonSlurper().parse(new ByteBufferInputStream(bytes));
301                                        case XML:
302                                                return new XmlSlurper().parse(new ByteBufferInputStream(bytes));
303                                        case TEXT:
304                                                return UTF_8.decode(bytes).toString();
305                                        default:
306                                                return bytes;
307                                }
308                        } catch (Exception e) {
309                                throw new IllegalStateException("Invalid " + this, e);
310                        }
311                }
312
313                public static ContentType forContentType(String contentType) {
314                        if (contentType.contains("json")) {
315                                return JSON;
316                        }
317                        if (contentType.contains("xml")) {
318                                return XML;
319                        }
320                        if (contentType.contains("text")) {
321                                return TEXT;
322                        }
323                        return BYTES;
324                }
325
326        }
327
328        public List<File> getSubtitles(File file, String... languages) throws Exception {
329                return getSubtitles(EMPTY_MAP, file, languages);
330        }
331
332        public List<File> getSubtitles(Map options, File file, String... languages) throws Exception {
333                List<File> subtitleFiles = new ArrayList<File>(languages.length);
334
335                // strict by default
336                boolean strict = optional(options, "strict", ScriptBaseClass::flag, true);
337
338                // accept any subtitle descriptor by default
339                Predicate<SubtitleDescriptor> filter = optional(options, "filter", ScriptBaseClass::selector, selectAll());
340
341                // SubRip is the only supported transcode target so the subtitle format is not configurable
342                SubtitleFormat format = SubtitleFormat.SubRip;
343
344                // ignore non-video files
345                if (VIDEO_FILES.accept(file) && file.length() > ONE_MEGABYTE) {
346                        // ignore video files that already have subtitles
347                        for (String language : languages) {
348                                File destination = new File(file.getParentFile(), SubtitleUtilities.formatSubtitle(getName(file), language, format.getFilter().extension()));
349                                if (destination.exists()) {
350                                        continue;
351                                }
352
353                                println("FIND " + destination.getName());
354                                SubtitleDescriptor descriptor = findSubtitles(file, language, strict, filter);
355                                if (descriptor == null) {
356                                        continue;
357                                }
358
359                                println("FETCH " + descriptor);
360                                MemoryFile subtitles = SubtitleUtilities.fetchSubtitle(descriptor);
361                                ByteBuffer bytes = SubtitleUtilities.exportSubtitles(subtitles, format, UTF_8);
362                                subtitleFiles.add(writeFile(bytes, destination));
363                        }
364                }
365
366                return subtitleFiles;
367        }
368
369        public SubtitleDescriptor findSubtitles(File file, String language, boolean strict, Predicate<SubtitleDescriptor> selector) throws Exception {
370                // resolve language code or language name
371                Locale locale = Language.forName(language).getLocale();
372
373                // lookup subtitles by hash
374                for (SubtitleLookupService service : WebServices.getSubtitleLookupServices(locale)) {
375                        SubtitleDescriptor descriptor = SubtitleUtilities.lookupSubtitlesByHash(service, singleton(file), locale, true, strict).values().stream().flatMap(List::stream).filter(selector).findFirst().orElse(null);
376                        if (descriptor != null) {
377                                return descriptor;
378                        }
379                }
380
381                // only lookup subtitles by hash in strict mode
382                if (strict) {
383                        return null;
384                }
385
386                // lookup subtitles by name
387                for (SubtitleProvider service : WebServices.getSubtitleProviders(locale)) {
388                        SubtitleDescriptor descriptor = SubtitleUtilities.findSubtitlesByName(service, singleton(file), locale, null, true, strict).values().stream().flatMap(List::stream).filter(selector).findFirst().orElse(null);
389                        if (descriptor != null) {
390                                return descriptor;
391                        }
392                }
393
394                return null;
395        }
396
397        private static <T> Predicate<T> selectAll() {
398                return object -> true;
399        }
400
401        private static <T> Predicate<T> selector(Object value) {
402                if (value instanceof Closure) {
403                        Closure closure = (Closure) value;
404                        return object -> DefaultTypeTransformation.castToBoolean(closure.call(object));
405                }
406                if (value instanceof Pattern) {
407                        Pattern pattern = (Pattern) value;
408                        return object -> pattern.matcher(object.toString()).find();
409                }
410                return null;
411        }
412
413        private static boolean flag(Object value) {
414                if (value instanceof Boolean) {
415                        return (Boolean) value;
416                }
417                if (value instanceof String) {
418                        return Boolean.parseBoolean((String) value);
419                }
420                return DefaultTypeTransformation.castToBoolean(value);
421        }
422
423        private static URL url(Object value) throws Exception {
424                if (value instanceof URL) {
425                        return (URL) value;
426                }
427                if (value instanceof URI) {
428                        return ((URI) value).toURL();
429                }
430                return WebRequest.newURL(value.toString());
431        }
432
433        private static <T> T optional(Map options, String name, Function<Object, T> value, T defaultValue) {
434                return Optional.ofNullable(options.get(name)).map(value).orElse(defaultValue);
435        };
436
437        public void NEWLINE(File file, Object c1, Object... cN) throws Exception {
438                if (file.length() > MemoryFile.LARGE_FILE_SIZE) {
439                        alert(format("NEWLINE file exists already: %s (%s)", file, formatSize(file.length())));
440                        return;
441                }
442
443                // check existing lines
444                List<String> current = file.exists() ? Files.readAllLines(file.toPath(), UTF_8) : emptyList();
445
446                // generate new lines
447                List<String> lines = list(this, c1, cN).stream().filter(v -> !isEmptyValue(this, v)).map(Object::toString).flatMap(NEWLINE::splitAsStream).map(String::trim).distinct().filter(line -> {
448                        // skip lines that already exist
449                        return !line.isEmpty() && !current.contains(line);
450                }).peek(line -> {
451                        // add feedback
452                        println(format("NEWLINE %s (%s)", line, file));
453                }).collect(toList());
454
455                // add the missing lines
456                if (lines.size() > 0) {
457                        Files.write(file.toPath(), lines, UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
458                }
459        }
460
461        public File XML(File file, Closure delegate) throws Exception {
462                return write(file, Builder.XML, delegate);
463        }
464
465        public String XML(Closure delegate) {
466                return Builder.XML.toString(delegate);
467        }
468
469        public File INI(File file, Closure delegate) throws Exception {
470                return write(file, Builder.INI, delegate);
471        }
472
473        public String INI(Closure delegate) throws Exception {
474                return Builder.INI.toString(delegate);
475        }
476
477        public File JSON(File file, Closure delegate) throws Exception {
478                return write(file, Builder.JSON, delegate);
479        }
480
481        public String JSON(Closure delegate) {
482                return Builder.JSON.toString(delegate);
483        }
484
485        private File write(File file, Builder builder, Closure delegate) throws Exception {
486                if (file.length() > MemoryFile.LARGE_FILE_SIZE) {
487                        alert(format("%s file exists already: %s (%s)", builder, file, formatSize(file.length())));
488                        return null;
489                }
490
491                byte[] bytes = builder.toString(delegate).getBytes(UTF_8);
492                println(format("%s %s (%s)", builder, file, formatSize(bytes.length)));
493
494                createFolders(file.getParentFile());
495                return writeFile(bytes, file);
496        }
497
498        private static Map<String, String> toStringMap(Map map) {
499                if (map.isEmpty()) {
500                        return EMPTY_MAP;
501                }
502
503                Map<String, String> properties = new LinkedHashMap<String, String>(map.size());
504                map.forEach((k, v) -> {
505                        if (k != null && v != null) {
506                                properties.put(k.toString(), v.toString());
507                        }
508                });
509                return properties;
510        }
511
512}