Adds a Letterbox cropper, and fixes cover images for the anime converter

Adds a new letterbox cropper, which will use 10 samples at 30 points in a video in order to find the correct crop to remove letter-boxing.
Makes sure to always copy codec of cover images, as ffmpeg treats them as video streams.

src/main/java/net/knarcraft/ffmpegconverter/FFMpegConvert.java

@@ -5,6 +5,7 @@ import net.knarcraft.ffmpegconverter.converter.AnimeConverter;
 import net.knarcraft.ffmpegconverter.converter.AudioConverter;
 import net.knarcraft.ffmpegconverter.converter.Converter;
 import net.knarcraft.ffmpegconverter.converter.DownScaleConverter;
+import net.knarcraft.ffmpegconverter.converter.LetterboxCropper;
 import net.knarcraft.ffmpegconverter.converter.MKVToMP4Transcoder;
 import net.knarcraft.ffmpegconverter.converter.MkvH264Converter;
 import net.knarcraft.ffmpegconverter.converter.MkvH265ReducedConverter;
@@ -96,7 +97,8 @@ public class FFMpegConvert {
                 8. DownScaleConverter
                 9. mp4 Subtitle Embed
                 10. Anime to h265 all streams
-                11. Stream reorder""", 1, 11);
+                11. Stream reorder
+                12. Letterbox cropper""", 1, 12);
 
         return switch (choice) {
             case 1 -> generateWebAnimeConverter();
@@ -110,6 +112,7 @@ public class FFMpegConvert {
             case 9 -> new SubtitleEmbed(FFPROBE_PATH, FFMPEG_PATH);
             case 10 -> generateAnimeConverter();
             case 11 -> generateStreamOrderConverter();
+            case 12 -> new LetterboxCropper(FFPROBE_PATH, FFMPEG_PATH);
             default -> null;
         };
     }

src/main/java/net/knarcraft/ffmpegconverter/container/FFMpegCommand.java

@@ -9,13 +9,13 @@ import java.util.List;
 /**
  * A class for generating and storing a ffmpeg command
  */
-public class FFMpegCommand {
+public class FFMpegCommand implements Cloneable {
 
-    private final @NotNull String executable;
-    private final @NotNull List<String> globalOptions;
-    private final @NotNull List<String> inputFileOptions;
-    private final @NotNull List<String> inputFiles;
-    private final @NotNull List<String> outputFileOptions;
+    private @NotNull String executable;
+    private @NotNull List<String> globalOptions;
+    private @NotNull List<String> inputFileOptions;
+    private @NotNull List<String> inputFiles;
+    private @NotNull List<String> outputFileOptions;
     private @Nullable String outputVideoCodec;
     private @NotNull String outputFile;
 
@@ -135,4 +135,21 @@ public class FFMpegCommand {
         return result.toArray(new String[0]);
     }
 
+    @Override
+    public FFMpegCommand clone() {
+        try {
+            FFMpegCommand clone = (FFMpegCommand) super.clone();
+            clone.outputVideoCodec = this.outputVideoCodec;
+            clone.outputFile = this.outputFile;
+            clone.executable = this.executable;
+            clone.globalOptions = new ArrayList<>(this.globalOptions);
+            clone.inputFileOptions = new ArrayList<>(this.inputFileOptions);
+            clone.inputFiles = new ArrayList<>(this.inputFiles);
+            clone.outputFileOptions = new ArrayList<>(this.outputFileOptions);
+            return clone;
+        } catch (CloneNotSupportedException e) {
+            throw new AssertionError();
+        }
+    }
+
 }

src/main/java/net/knarcraft/ffmpegconverter/converter/AnimeConverter.java

@@ -28,6 +28,7 @@ import net.knarcraft.ffmpegconverter.converter.sorter.SubtitleLanguageSorter;
 import net.knarcraft.ffmpegconverter.converter.sorter.SubtitleTitleSorter;
 import net.knarcraft.ffmpegconverter.property.MinimalSubtitlePreference;
 import net.knarcraft.ffmpegconverter.streams.AudioStream;
+import net.knarcraft.ffmpegconverter.streams.OtherStream;
 import net.knarcraft.ffmpegconverter.streams.SubtitleStream;
 import net.knarcraft.ffmpegconverter.streams.VideoStream;
 import net.knarcraft.ffmpegconverter.utility.FFMpegHelper;
@@ -164,6 +165,14 @@ public class AnimeConverter extends AbstractConverter {
         modules.add(new SetOutputFileModule(outFile));
 
         new ModuleExecutor(command, modules).execute();
+
+        int index = videoStreams.size();
+        for (OtherStream stream : probeResult.getOtherStreams()) {
+            if (stream.isCoverImage()) {
+                command.addOutputFileOption("-c:v:" + index++, "copy");
+            }
+        }
+
         return command;
     }
 

src/main/java/net/knarcraft/ffmpegconverter/converter/LetterboxCropper.java

@@ -0,0 +1,197 @@
+package net.knarcraft.ffmpegconverter.converter;
+
+import net.knarcraft.ffmpegconverter.container.FFMpegCommand;
+import net.knarcraft.ffmpegconverter.container.ProcessResult;
+import net.knarcraft.ffmpegconverter.container.StreamProbeResult;
+import net.knarcraft.ffmpegconverter.converter.module.ConverterModule;
+import net.knarcraft.ffmpegconverter.converter.module.DebugModule;
+import net.knarcraft.ffmpegconverter.converter.module.ModuleExecutor;
+import net.knarcraft.ffmpegconverter.converter.module.mapping.MapAllModule;
+import net.knarcraft.ffmpegconverter.converter.module.output.CopyAudioModule;
+import net.knarcraft.ffmpegconverter.converter.module.output.CopySubtitlesModule;
+import net.knarcraft.ffmpegconverter.converter.module.output.SetOutputFileModule;
+import net.knarcraft.ffmpegconverter.streams.AudioStream;
+import net.knarcraft.ffmpegconverter.streams.StreamObject;
+import net.knarcraft.ffmpegconverter.utility.FFMpegHelper;
+import net.knarcraft.ffmpegconverter.utility.StringUtil;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A converter that crops letter-boxing (black padding) from video files
+ */
+public class LetterboxCropper extends AbstractConverter {
+
+    private final static String SPACER = "|,-,|";
+
+    /**
+     * Initializes a new letter box cropper
+     *
+     * @param ffprobePath <p>Path/command to ffprobe</p>
+     * @param ffmpegPath  <p>Path/command to ffmpeg</p>
+     */
+    public LetterboxCropper(@NotNull String ffprobePath, @NotNull String ffmpegPath) {
+        super("mkv");
+        this.ffprobePath = ffprobePath;
+        this.ffmpegPath = ffmpegPath;
+    }
+
+    @Override
+    public @NotNull List<String> getValidFormats() {
+        return List.of("mkv", "mp4");
+    }
+
+    @Override
+    @Nullable
+    public FFMpegCommand generateConversionCommand(@NotNull String executable, @NotNull StreamProbeResult probeResult,
+                                                   @NotNull String outFile) {
+        File inputFile = probeResult.parsedFiles().get(0);
+
+        List<ConverterModule> modules = new ArrayList<>();
+
+        Map<String, Integer> cropValues = getLetterboxCropValues(modules, inputFile);
+
+        FFMpegCommand convertCommand = getConversionCommand(cropValues, inputFile);
+
+        modules.add(new MapAllModule<>(probeResult.getVideoStreams()));
+
+        // Map audio if present
+        List<AudioStream> audioStreams = probeResult.getAudioStreams();
+        if (!audioStreams.isEmpty()) {
+            modules.add(new MapAllModule<>(audioStreams));
+            setOutputIndexes(audioStreams);
+            modules.add(new CopyAudioModule(audioStreams));
+        }
+
+        // Map subtitles if present
+        List<StreamObject> subtitleStreams = new ArrayList<>(probeResult.getSubtitleStreams());
+        if (!subtitleStreams.isEmpty()) {
+            modules.add(new MapAllModule<>(probeResult.getSubtitleStreams()));
+            setOutputIndexes(subtitleStreams);
+            modules.add(new CopySubtitlesModule());
+        }
+
+        // Map all attached streams, such as fonts and covers
+        modules.add(new MapAllModule<>(probeResult.getOtherStreams()));
+
+        modules.add(new SetOutputFileModule(outFile));
+
+        new ModuleExecutor(convertCommand, modules).execute();
+        return convertCommand;
+    }
+
+    /**
+     * Gets the initial command used for removing the letterbox from a video file
+     *
+     * @param cropValues <p>The possible crop values to be used</p>
+     * @param inputFile  <p>The file to be converted</p>
+     * @return <p>The initial command to use for converting the file</p>
+     */
+    @NotNull
+    private FFMpegCommand getConversionCommand(@NotNull Map<String, Integer> cropValues, @NotNull File inputFile) {
+        String crop = null;
+        int counts = 0;
+
+        for (Map.Entry<String, Integer> entry : cropValues.entrySet()) {
+            if (crop == null || entry.getValue() > counts) {
+                crop = entry.getKey();
+                counts = entry.getValue();
+            }
+        }
+
+        if (crop == null) {
+            throw new IllegalArgumentException("Unable to detect letterbox for video");
+        }
+
+        FFMpegCommand convertCommand = new FFMpegCommand(ffmpegPath);
+
+        convertCommand.addInputFileOption(inputFile.getName());
+        convertCommand.addOutputFileOption("-vf", "crop=" + crop);
+        convertCommand.addOutputFileOption("-c:v", "libx265");
+        convertCommand.addOutputFileOption("-crf", "22");
+        convertCommand.addOutputFileOption("-preset", "medium");
+        convertCommand.addOutputFileOption("-tune", "ssim");
+        convertCommand.addOutputFileOption("-profile:v", "main10");
+        convertCommand.addOutputFileOption("-level:v", "4");
+        convertCommand.addOutputFileOption("-x265-params", "bframes=8:scenecut-bias=0.05:me=star:subme=5:" +
+                "refine-mv=1:limit-refs=1:rskip=0:max-merge=5:rc-lookahead=80:lookahead-slices=5:min-keyint=23:" +
+                "max-luma=1023:psy-rd=2:strong-intra-smoothing=0:b-intra=1:hist-threshold=0.03");
+        return convertCommand;
+    }
+
+    /**
+     * Gets a map counting each time a given letterbox crop suggestion has been encountered
+     *
+     * <p>As some parts of a video might be entirely black, or some parts of a video might be dark enough that parts of
+     * a scene might be detected as part of the letterbox, this takes 30 crop-detect samples evenly spread among the
+     * file. It can be assumed that the crop-detect value that was encountered the most times is the correct one.</p>
+     *
+     * @param modules   <p>The converter modules to append to</p>
+     * @param inputFile <p>The input file to find letterbox of</p>
+     * @return <p>A map counting all returned crop-detect values</p>
+     */
+    @NotNull
+    private Map<String, Integer> getLetterboxCropValues(@NotNull List<ConverterModule> modules, @NotNull File inputFile) {
+        Map<String, Integer> cropValues = new HashMap<>();
+
+        if (this.debug) {
+            modules.add(new DebugModule(0, 0));
+        }
+
+        FFMpegCommand probeCommand = new FFMpegCommand(ffmpegPath);
+        probeCommand.addGlobalOption("-nostats", "-hide_banner");
+        probeCommand.addInputFile(inputFile.toString());
+        probeCommand.addOutputFileOption("-vframes", "10");
+        probeCommand.addOutputFileOption("-vf", "cropdetect");
+        probeCommand.addOutputFileOption("-f", "null");
+        probeCommand.addOutputFileOption("-");
+
+        double duration;
+        try {
+            duration = FFMpegHelper.getDuration(ffprobePath, inputFile);
+        } catch (IOException | NumberFormatException exception) {
+            throw new RuntimeException("Unable to get duration from video file");
+        }
+
+        int increments = (int) (duration / 30d);
+
+        for (int i = 0; i < duration; i += increments) {
+            FFMpegCommand clone = probeCommand.clone();
+            clone.addInputFileOption("-ss", String.valueOf(i));
+
+            ProcessBuilder processBuilder = new ProcessBuilder(probeCommand.getResult());
+            ProcessResult result = null;
+            try {
+                result = FFMpegHelper.runProcess(processBuilder, inputFile.getParentFile(), SPACER, false);
+            } catch (IOException ignored) {
+            }
+            if (result == null || result.exitCode() != 0) {
+                throw new IllegalArgumentException("File probe failed with code " + (result == null ? "null" :
+                        result.exitCode()));
+            }
+
+            List<String> parsed = StringUtil.stringBetween(result.output(), "crop=", SPACER);
+            for (String string : parsed) {
+                if (string.contains("-")) {
+                    continue;
+                }
+
+                if (cropValues.containsKey(string)) {
+                    cropValues.put(string, cropValues.get(string) + 1);
+                } else {
+                    cropValues.put(string, 1);
+                }
+            }
+        }
+
+        return cropValues;
+    }
+
+}

src/main/java/net/knarcraft/ffmpegconverter/converter/MkvH264Converter.java

@@ -32,7 +32,7 @@ public class MkvH264Converter extends AbstractConverter {
      * @param ffprobePath <p>Path/command to ffprobe.</p>
      * @param ffmpegPath  <p>Path/command to ffmpeg.</p>
      */
-    public MkvH264Converter(String ffprobePath, String ffmpegPath) {
+    public MkvH264Converter(@NotNull String ffprobePath, @NotNull String ffmpegPath) {
         super("mkv");
         this.ffprobePath = ffprobePath;
         this.ffmpegPath = ffmpegPath;
@@ -76,6 +76,7 @@ public class MkvH264Converter extends AbstractConverter {
         List<StreamObject> subtitleStreams = new ArrayList<>(probeResult.getSubtitleStreams());
         if (!subtitleStreams.isEmpty()) {
             modules.add(new MapAllModule<>(probeResult.getSubtitleStreams()));
+            setOutputIndexes(subtitleStreams);
             modules.add(new CopySubtitlesModule());
         }
 

src/main/java/net/knarcraft/ffmpegconverter/property/StreamType.java

@@ -20,6 +20,13 @@ public enum StreamType {
      */
     SUBTITLE,
 
+    /**
+     * A cover image
+     *
+     * <p>Cover images are treated as video streams by ffmpeg, so they need special treatment</p>
+     */
+    COVER_IMAGE,
+
     /**
      * None of the above
      */

src/main/java/net/knarcraft/ffmpegconverter/streams/AudioStream.java

@@ -18,7 +18,7 @@ public class AudioStream extends AbstractStream implements StreamObject {
      *
      * @param streamInfo    <p>All info about the stream</p>
      * @param inputIndex    <p>The index of the input file containing this stream</p>
-     * @param relativeIndex <p>The index of the audio stream relative to other audio streams.</p>
+     * @param relativeIndex <p>The index of the audio stream relative to other audio streams</p>
      */
     public AudioStream(@NotNull Map<StreamTag, String> streamInfo, int inputIndex, int relativeIndex) {
         super(streamInfo, inputIndex, relativeIndex);

src/main/java/net/knarcraft/ffmpegconverter/streams/OtherStream.java

@@ -9,14 +9,27 @@ import java.util.Map;
  */
 public class OtherStream extends AbstractStream {
 
+    private final boolean isCoverImage;
+
     /**
      * Instantiates a new other stream
      *
      * @param streamInfo   <p>All info about the stream</p>
      * @param inputIndex   <p>The index of the input file this stream belongs to</p>
+     * @param isCoverImage <p>Whether this stream is a cover image stream</p>
      */
-    public OtherStream(@NotNull Map<StreamTag, String> streamInfo, int inputIndex) {
+    public OtherStream(@NotNull Map<StreamTag, String> streamInfo, int inputIndex, boolean isCoverImage) {
         super(streamInfo, inputIndex, 0);
+        this.isCoverImage = isCoverImage;
+    }
+
+    /**
+     * Gets whether this stream is a cover image stream
+     *
+     * @return <p>True if this stream is a cover image stream</p>
+     */
+    public boolean isCoverImage() {
+        return isCoverImage;
     }
 
     @Override

src/main/java/net/knarcraft/ffmpegconverter/streams/StreamTag.java

@@ -455,6 +455,16 @@ public enum StreamTag {
      * <p>Applicable for all 3 stream types</p>
      */
     TAG_DURATION("TAG:DURATION"),
+
+    /**
+     * The file name of the attached file (image/font)
+     */
+    TAG_FILE_NAME("TAG:filename"),
+
+    /**
+     * The mime type of the attached file (image/font)
+     */
+    TAG_MIME_TYPE("TAG:mimetype"),
     ;
 
     private static final Map<String, StreamTag> tagLookup = new HashMap<>();

src/main/java/net/knarcraft/ffmpegconverter/streams/SubtitleStream.java

@@ -17,7 +17,7 @@ public class SubtitleStream extends AbstractStream implements StreamObject {
      *
      * @param streamInfo    <p>All info about the stream</p>
      * @param inputIndex    <p>The index of the input file containing this stream</p>
-     * @param relativeIndex <p>The index of the subtitle stream relative to other subtitle streams.</p>
+     * @param relativeIndex <p>The index of the subtitle stream relative to other subtitle streams</p>
      */
     public SubtitleStream(@NotNull Map<StreamTag, String> streamInfo, int inputIndex, int relativeIndex) {
         super(streamInfo, inputIndex, relativeIndex);

src/main/java/net/knarcraft/ffmpegconverter/streams/VideoStream.java

@@ -18,7 +18,7 @@ public class VideoStream extends AbstractStream implements StreamObject {
      *
      * @param streamInfo    <p>All info about the stream</p>
      * @param inputIndex    <p>The index of the input file containing this stream</p>
-     * @param relativeIndex <p>The index of the video stream relative to other video streams.</p>
+     * @param relativeIndex <p>The index of the video stream relative to other video streams</p>
      */
     public VideoStream(@NotNull Map<StreamTag, String> streamInfo, int inputIndex, int relativeIndex) {
         super(streamInfo, inputIndex, relativeIndex);

src/main/java/net/knarcraft/ffmpegconverter/utility/FFMpegHelper.java

@@ -34,40 +34,6 @@ public final class FFMpegHelper {
 
     }
 
-    /*
-    Developer notes:
-      Setting the default track:
-        -disposition:s:0 +default
-        Where s,a,v is the type specifier, and the number is the relative index
-        
-        To unset default:
-        -disposition:s:0 -default
-    
-      -map command for reordering:
-          First number is the index of the input file, which is 0, unless more files are given as input
-          Optionally, a,v,s can be used to select the type of stream to map
-          Lastly, the number is either the global index of the stream, or the relative, if a type has been specified
-          If only the first number is given, all streams from that file are selected
-          The output file will contain all mapped streams in the order they are mapped
-          
-          Plan: Sort all streams by set criteria. Map all selected streams by looping using their relative index for
-          selection.
-       
-      Streams should probably have an input index, so it's easier to treat extra subtitles more seamlessly. So, by
-      including any external subtitle files as input, there would be no need to fiddle more with storing input files.
-      
-      Instead of storing the ffmpeg command as a list of strings, it should be stored as an object with different list
-      for input arguments and output arguments. That way, it would be much easier to add input-related arguments later
-      in the process.
-      
-      ffmpeg -codecs:
-        Used for checking which codecs are available.
-      
-      ffmpeg -h encoder=h264_nvenc:
-        Used to see available encoder presets/profiles/info
-      
-     */
-
     /**
      * Gets streams from a file
      *
@@ -280,6 +246,29 @@ public final class FFMpegHelper {
         return StringUtil.stringBetween(result.output(), "[STREAM]", "[/STREAM]");
     }
 
+    /**
+     * Gets the duration, in seconds, of the given file
+     *
+     * @param ffprobePath <p>The path to the ffprobe executable</p>
+     * @param file        <p>The file to get the duration of</p>
+     * @return <p>The duration</p>
+     * @throws IOException           <p>If unable to probe the file</p>
+     * @throws NumberFormatException <p>If ffmpeg returns a non-number</p>
+     */
+    public static double getDuration(@NotNull String ffprobePath, @NotNull File file) throws IOException, NumberFormatException {
+        FFMpegCommand probeCommand = new FFMpegCommand(ffprobePath);
+        probeCommand.addGlobalOption("-v", "error", "-show_entries", "format=duration", "-of",
+                "default=noprint_wrappers=1:nokey=1");
+        probeCommand.addInputFile(file.toString());
+
+        ProcessBuilder processBuilder = new ProcessBuilder(probeCommand.getResult());
+        ProcessResult result = runProcess(processBuilder, file.getParentFile(), "", false);
+        if (result.exitCode() != 0) {
+            throw new IllegalArgumentException("File probe failed with code " + result.exitCode());
+        }
+        return Double.parseDouble(result.output().trim());
+    }
+
     /**
      * Takes a list of all streams and parses each stream into one of three objects
      *
@@ -294,7 +283,8 @@ public final class FFMpegHelper {
     private static StreamProbeResult parseStreams(@NotNull String ffprobePath, @NotNull List<String> streams,
                                                   @NotNull File file, @NotNull List<String> subtitleFormats,
                                                   @NotNull List<String> audioFormats) throws IOException {
-        StreamProbeResult probeResult = new StreamProbeResult(new ArrayList<>(List.of(file)), parseStreamObjects(streams));
+        StreamProbeResult probeResult = new StreamProbeResult(new ArrayList<>(List.of(file)),
+                parseStreamObjects(streams));
         getExternalStreams(probeResult, ffprobePath, file.getParentFile(), file.getName(), subtitleFormats);
         getExternalStreams(probeResult, ffprobePath, file.getParentFile(), file.getName(), audioFormats);
         return probeResult;
@@ -319,18 +309,11 @@ public final class FFMpegHelper {
             StreamType streamType = getStreamType(streamInfo);
 
             switch (streamType) {
-                case VIDEO:
-                    parsedStreams.add(new VideoStream(streamInfo, 0, relativeVideoIndex++));
-                    break;
-                case AUDIO:
-                    parsedStreams.add(new AudioStream(streamInfo, 0, relativeAudioIndex++));
-                    break;
-                case SUBTITLE:
-                    parsedStreams.add(new SubtitleStream(streamInfo, 0, relativeSubtitleIndex++));
-                    break;
-                case OTHER:
-                    parsedStreams.add(new OtherStream(streamInfo, 0));
-                    break;
+                case VIDEO -> parsedStreams.add(new VideoStream(streamInfo, 0, relativeVideoIndex++));
+                case AUDIO -> parsedStreams.add(new AudioStream(streamInfo, 0, relativeAudioIndex++));
+                case SUBTITLE -> parsedStreams.add(new SubtitleStream(streamInfo, 0, relativeSubtitleIndex++));
+                case OTHER -> parsedStreams.add(new OtherStream(streamInfo, 0, false));
+                case COVER_IMAGE -> parsedStreams.add(new OtherStream(streamInfo, 0, true));
             }
         }
         return parsedStreams;
@@ -347,11 +330,13 @@ public final class FFMpegHelper {
         String codecType = ValueParsingHelper.parseString(streamInfo.get(StreamTag.CODEC_TYPE), "");
         switch (codecType) {
             case "video":
+                String mime = ValueParsingHelper.parseString(streamInfo.get(StreamTag.TAG_MIME_TYPE), "");
                 // Some attached covers are marked as video streams
-                if (ValueParsingHelper.parseInt(streamInfo.get(StreamTag.DISPOSITION_ATTACHED_PIC), 0) != 1) {
+                if (ValueParsingHelper.parseInt(streamInfo.get(StreamTag.DISPOSITION_ATTACHED_PIC), 0) != 1 &&
+                        !mime.startsWith("image/") && !mime.endsWith("-font")) {
                     return StreamType.VIDEO;
                 } else {
-                    return StreamType.OTHER;
+                    return StreamType.COVER_IMAGE;
                 }
             case "audio":
                 return StreamType.AUDIO;

src/main/java/net/knarcraft/ffmpegconverter/utility/StringUtil.java

@@ -8,7 +8,7 @@ import java.util.List;
 /**
  * A class which helps with operations on strings
  */
-final class StringUtil {
+public final class StringUtil {
 
     private StringUtil() {