FlatFileDataProcessor will handle fixing and repairing the data

src/main/java/com/gmail/nossr50/database/FlatFileDataContainer.java

@@ -1,5 +0,0 @@
-package com.gmail.nossr50.database;
-
-//Marker interface
-public interface FlatFileDataContainer {
-}

src/main/java/com/gmail/nossr50/database/FlatFileDataFlag.java

@@ -4,10 +4,9 @@ public enum FlatFileDataFlag {
     INCOMPLETE,
     BAD_VALUES,
     MISSING_NAME,
-    DUPLICATE_NAME_FIXABLE,
+    DUPLICATE_NAME,
-    DUPLICATE_NAME_NOT_FIXABLE,
     DUPLICATE_UUID,
-    MISSING_OR_NULL_UUID,
+    BAD_UUID_DATA, //Can be because it is missing, null, or just not compatible data
     TOO_INCOMPLETE,
     JUNK,
     EMPTY_LINE,

src/main/java/com/gmail/nossr50/database/FlatFileDataProcessor.java

@@ -1,30 +1,27 @@
 package com.gmail.nossr50.database;
 
-import com.gmail.nossr50.database.flatfile.CategorizedFlatFileData;
+import com.gmail.nossr50.database.flatfile.FlatFileDataBuilder;
-import com.gmail.nossr50.database.flatfile.CategorizedFlatFileDataBuilder;
+import com.gmail.nossr50.database.flatfile.FlatFileDataContainer;
+import com.gmail.nossr50.database.flatfile.FlatFileSaveDataProcessor;
 import org.jetbrains.annotations.NotNull;
 
-import java.io.File;
 import java.util.*;
 import java.util.logging.Logger;
 
 import static com.gmail.nossr50.database.FlatFileDatabaseManager.*;
 
 public class FlatFileDataProcessor {
-    public static final String INVALID_OLD_USERNAME = "_INVALID_OLD_USERNAME_";
+    private final @NotNull List<FlatFileDataContainer> flatFileDataContainers;
-    private @NotNull List<CategorizedFlatFileData> categorizedDataList;
+    private final @NotNull List<FlatFileDataFlag> flatFileDataFlags;
-    private @NotNull List<FlatFileDataFlag> flatFileDataFlags;
-    private final @NotNull File userFile;
     private final @NotNull Logger logger;
     private final HashSet<String> names;
     private final HashSet<UUID> uuids;
     private int uniqueProcessingID;
     boolean corruptDataFound;
 
-    public FlatFileDataProcessor(@NotNull File userFile, @NotNull Logger logger) {
+    public FlatFileDataProcessor(@NotNull Logger logger) {
-        this.userFile = userFile;
         this.logger = logger;
-        categorizedDataList = new ArrayList<>();
+        flatFileDataContainers = new ArrayList<>();
         flatFileDataFlags = new ArrayList<>();
         names = new HashSet<>();
         uuids = new HashSet<>();
@@ -32,16 +29,7 @@ public class FlatFileDataProcessor {
     }
 
     public void processData(@NotNull String lineData) {
-        CategorizedFlatFileDataBuilder builder = new CategorizedFlatFileDataBuilder(lineData, uniqueProcessingID);
+        assert !lineData.isEmpty();
-        uniqueProcessingID++;
-
-        /*
-         * Is the line empty?
-         */
-        if (lineData.isEmpty()) {
-            registerData(builder.appendFlag(FlatFileDataFlag.EMPTY_LINE));
-            return;
-        }
 
         //Make sure the data line is "correct"
         if(lineData.charAt(lineData.length() - 1) != ':') {
@@ -53,6 +41,11 @@ public class FlatFileDataProcessor {
         //Split the data into an array
         String[] splitDataLine = lineData.split(":");
 
+        FlatFileDataBuilder builder = new FlatFileDataBuilder(splitDataLine, uniqueProcessingID);
+        uniqueProcessingID++;
+        boolean[] badDataValues = new boolean[DATA_ENTRY_COUNT];
+        boolean anyBadData = false;
+
         //This is the minimum size of the split array needed to be considered proper data
         if(splitDataLine.length < getMinimumSplitDataLength()) {
             //Data is considered junk
@@ -82,7 +75,6 @@ public class FlatFileDataProcessor {
          * Check for duplicate names
          */
 
-        boolean nameIsDupe = false;
         boolean invalidUUID = false;
 
         String name = splitDataLine[USERNAME_INDEX];
@@ -91,12 +83,17 @@ public class FlatFileDataProcessor {
         if(name.isEmpty()) {
             reportBadDataLine("No name found for data", "[MISSING NAME]", lineData);
             builder.appendFlag(FlatFileDataFlag.MISSING_NAME);
+            anyBadData = true;
+            badDataValues[USERNAME_INDEX] = true;
         }
 
         if(strOfUUID.isEmpty() || strOfUUID.equalsIgnoreCase("NULL")) {
             invalidUUID = true;
+            badDataValues[UUID_INDEX] = true;
             reportBadDataLine("Empty/null UUID for user", "Empty/null", lineData);
-            builder.appendFlag(FlatFileDataFlag.MISSING_OR_NULL_UUID);
+            builder.appendFlag(FlatFileDataFlag.BAD_UUID_DATA);
+
+            anyBadData = true;
         }
 
         UUID uuid = null;
@@ -104,36 +101,23 @@ public class FlatFileDataProcessor {
         try {
             uuid = UUID.fromString(strOfUUID);
         } catch (IllegalArgumentException e) {
-            invalidUUID = true;
             //UUID does not conform
-
+            invalidUUID = true;
+            badDataValues[UUID_INDEX] = true;
             reportBadDataLine("Invalid UUID data found for user", strOfUUID, lineData);
-            e.printStackTrace();
+            builder.appendFlag(FlatFileDataFlag.BAD_UUID_DATA);
         }
 
         //Duplicate UUID is no good, reject them
-        if(uuid != null && uuids.contains(uuid)) {
+        if(!invalidUUID && uuid != null && uuids.contains(uuid)) {
             registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_UUID));
             return;
         }
 
         uuids.add(uuid);
 
-
         if(names.contains(name)) {
-            //Duplicate entry
+            builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME);
-            nameIsDupe = true;
-
-            //We can accept them if they are a duped name if they have a unique UUID
-            if(invalidUUID) {
-                //Reject the data
-                reportBadDataLine("Duplicate user found and due to a missing UUID their data had to be discarded", name, lineData);
-
-                registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_NOT_FIXABLE));
-                return;
-            } else {
-                builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_FIXABLE);
-            }
         }
 
         if(!name.isEmpty())
@@ -141,10 +125,17 @@ public class FlatFileDataProcessor {
 
         //Make sure the data is up to date schema wise
         if(splitDataLine.length < DATA_ENTRY_COUNT) {
-            splitDataLine = Arrays.copyOf(splitDataLine, DATA_ENTRY_COUNT+1);
+            int oldLength = splitDataLine.length;
-            lineData = org.apache.commons.lang.StringUtils.join(splitDataLine, ":") + ":";
+            splitDataLine = Arrays.copyOf(splitDataLine, DATA_ENTRY_COUNT);
+            int newLength = splitDataLine.length;
+
+            //TODO: Test this
+            for(int i = oldLength; i < (newLength - 1); i++){
+                badDataValues[i] = true;
+            }
+
             builder.appendFlag(FlatFileDataFlag.INCOMPLETE);
-            builder.setStringDataRepresentation(lineData);
+            builder.setSplitStringData(splitDataLine);
         }
 
         /*
@@ -153,9 +144,6 @@ public class FlatFileDataProcessor {
          */
 
         //Check each data for bad values
-        boolean[] badDataValues = new boolean[DATA_ENTRY_COUNT];
-        boolean anyBadData = false;
-
         for(int i = 0; i < DATA_ENTRY_COUNT; i++) {
             if(shouldNotBeEmpty(splitDataLine[i], i)) {
                 badDataValues[i] = true;
@@ -175,6 +163,7 @@ public class FlatFileDataProcessor {
 
         if(anyBadData) {
             builder.appendFlag(FlatFileDataFlag.BAD_VALUES);
+            builder.appendBadDataValues(badDataValues);
         }
 
         registerData(builder);
@@ -240,13 +229,15 @@ public class FlatFileDataProcessor {
         return UUID_INDEX + 1;
     }
 
-    private void registerData(@NotNull CategorizedFlatFileDataBuilder builder) {
+    private void registerData(@NotNull FlatFileDataBuilder builder) {
-        CategorizedFlatFileData categorizedFlatFileData = builder.build();
+        FlatFileDataContainer flatFileDataContainer = builder.build();
-        categorizedDataList.add(categorizedFlatFileData);
+        flatFileDataContainers.add(flatFileDataContainer);
-        flatFileDataFlags.addAll(categorizedFlatFileData.getDataFlags());
+
+        if(flatFileDataContainer.getDataFlags() != null)
+            flatFileDataFlags.addAll(flatFileDataContainer.getDataFlags());
     }
 
-    public @NotNull ExpectedType getExpectedValueType(int dataIndex) {
+    public @NotNull ExpectedType getExpectedValueType(int dataIndex) throws IndexOutOfBoundsException {
         switch(dataIndex) {
             case USERNAME_INDEX:
                 return ExpectedType.STRING;
@@ -297,13 +288,13 @@ public class FlatFileDataProcessor {
                 return ExpectedType.FLOAT;
             case UUID_INDEX:
                 return ExpectedType.UUID;
-            default:
-                return ExpectedType.OUT_OF_RANGE;
         }
+
+        throw new IndexOutOfBoundsException();
     }
 
-    public @NotNull List<CategorizedFlatFileData> getCategorizedDataList() {
+    public @NotNull List<FlatFileDataContainer> getFlatFileDataContainers() {
-        return categorizedDataList;
+        return flatFileDataContainers;
     }
 
     public @NotNull List<FlatFileDataFlag> getFlatFileDataFlags() {
@@ -313,4 +304,23 @@ public class FlatFileDataProcessor {
     public int getDataFlagCount() {
         return flatFileDataFlags.size();
     }
+
+    public @NotNull StringBuilder processDataForSave() {
+        StringBuilder stringBuilder = new StringBuilder();
+
+        //Fix our data if needed and prepare it to be saved
+
+        for(FlatFileDataContainer dataContainer : flatFileDataContainers) {
+            String[] splitData = FlatFileSaveDataProcessor.getPreparedSaveDataLine(dataContainer);
+
+            if(splitData == null)
+                continue;
+
+            String fromSplit = org.apache.commons.lang.StringUtils.join(splitData, ":") + ":";
+            stringBuilder.append(fromSplit).append("\r\n");
+        }
+
+        return stringBuilder;
+    }
+
 }

src/main/java/com/gmail/nossr50/database/FlatFileDatabaseManager.java

@@ -20,6 +20,7 @@ import java.util.logging.Logger;
 
 public final class FlatFileDatabaseManager implements DatabaseManager {
     public static final String IGNORED = "IGNORED";
+    public static final String LEGACY_INVALID_OLD_USERNAME = "_INVALID_OLD_USERNAME_'";
     private final @NotNull EnumMap<PrimarySkillType, List<PlayerStat>> playerStatHash = new EnumMap<PrimarySkillType, List<PlayerStat>>(PrimarySkillType.class);
     private final @NotNull List<PlayerStat> powerLevels = new ArrayList<>();
     private long lastUpdate = 0;
@@ -965,24 +966,45 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         playerStatHash.put(PrimarySkillType.ALCHEMY, alchemy);
     }
 
+    /**
+     * Makes sure that the users file has valid entries
+     * @return
+     */
     public @Nullable List<FlatFileDataFlag> checkFileHealthAndStructure() {
         FlatFileDataProcessor dataProcessor = null;
 
         if (usersFile.exists()) {
             BufferedReader bufferedReader = null;
+            FileWriter fileWriter = null;
 
             synchronized (fileWritingLock) {
 
-                dataProcessor = new FlatFileDataProcessor(usersFile, logger);
+                dataProcessor = new FlatFileDataProcessor(logger);
 
                 try {
                     String currentLine;
                     bufferedReader = new BufferedReader(new FileReader(usersFilePath));
+
+                    //Analyze the data
                     while ((currentLine = bufferedReader.readLine()) != null) {
+                        if(currentLine.isEmpty())
+                            continue;
+
+                        //TODO: We are never passing empty lines, should we remove the flag for them?
                         dataProcessor.processData(currentLine);
                     }
+
+                    //Only update the file if needed
+                    if(dataProcessor.getFlatFileDataFlags().size() > 0) {
+                        logger.info("Saving the updated and or repaired FlatFile Database...");
+                        fileWriter = new FileWriter(usersFilePath);
+                        //Write data to file
+                        fileWriter.write(dataProcessor.processDataForSave().toString());
+                    }
                 } catch (IOException e) {
                     e.printStackTrace();
+                } finally {
+                    closeResources(bufferedReader, fileWriter);
                 }
             }
         }
@@ -994,139 +1016,23 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         }
     }
 
-
+    private void closeResources(BufferedReader bufferedReader, FileWriter fileWriter) {
-    /**
+        if(bufferedReader != null) {
-     * Checks that the file is present and valid
+            try {
-     */
+                bufferedReader.close();
-    public int checkFileHealthAndStructureOld() {
+            }
-        boolean corruptDataFound = false;
+            catch (IOException e) {
-        boolean oldDataFound = false;
+                e.printStackTrace();
-
-        if (usersFile.exists()) {
-            BufferedReader in = null;
-            FileWriter out = null;
-
-            synchronized (fileWritingLock) {
-                try {
-
-                    in = new BufferedReader(new FileReader(usersFilePath));
-                    StringBuilder writer = new StringBuilder();
-                    String line;
-                    HashSet<String> usernames = new HashSet<>();
-                    HashSet<String> players = new HashSet<>();
-
-                    while ((line = in.readLine()) != null) {
-                        // Remove empty lines from the file
-                        if (line.isEmpty()) {
-                            continue;
-                        }
-
-                        // Length checks depend on last rawSplitData being ':'
-                        if (line.charAt(line.length() - 1) != ':') {
-                            line = line.concat(":");
-                        }
-
-                        String[] rawSplitData = line.split(":");
-
-                        //Not enough data found to be considered a user reliably (NOTE: not foolproof)
-                        if(rawSplitData.length < (UUID_INDEX + 1)) {
-                            if(!corruptDataFound) {
-                                logger.severe("Some corrupt data was found in mcmmo.users and has been repaired, it is possible that some player data has been lost in this process.");
-                                corruptDataFound = true;
-                            }
-
-                            if(rawSplitData.length >= 10 //The value here is kind of arbitrary, it shouldn't be too low to avoid false positives, but also we aren't really going to correctly identify when player data has been corrupted or not with 100% accuracy ever
-                                    && rawSplitData[0] != null && !rawSplitData[0].isEmpty()) {
-                                if(rawSplitData[0].length() <= 16 && rawSplitData[0].length() >= 3) {
-                                    logger.severe("Not enough data found to recover corrupted player data for user: "+rawSplitData[0]);
-                                }
-                            }
-                            //This user may have had a name so declare it
-
-                            continue;
-                        }
-
-                        // Prevent the same username from being present multiple times
-                        if (!usernames.add(rawSplitData[USERNAME_INDEX])) {
-                            //TODO: Check if the commented out code was even necessary
-                            rawSplitData[USERNAME_INDEX] = "_INVALID_OLD_USERNAME_'";
-                            if (rawSplitData.length < UUID_INDEX + 1 || rawSplitData[UUID_INDEX].equals("NULL")) {
-                                logger.severe("Fixing duplicate player names found in mcmmo.users");
-                                continue;
-                            }
-                        }
-
-                        // Prevent the same player from being present multiple times
-                        if (rawSplitData.length >= (UUID_INDEX + 1) //TODO: Test this condition
-                                && (!rawSplitData[UUID_INDEX].isEmpty()
-                                && !rawSplitData[UUID_INDEX].equals("NULL") && !players.add(rawSplitData[UUID_INDEX]))) {
-
-                            logger.severe("Removing duplicate player data from mcmmo.users");
-                            logger.info("Duplicate Data: "+line);
-                            continue;
-                        }
-
-                        //Correctly size the data (null entries for missing values)
-                        if(line.length() < DATA_ENTRY_COUNT) { //TODO: Test this condition
-                            oldDataFound = true;
-                            String[] correctSizeSplitData = Arrays.copyOf(rawSplitData, DATA_ENTRY_COUNT);
-                            line = org.apache.commons.lang.StringUtils.join(correctSizeSplitData, ":") + ":";
-                            rawSplitData = line.split(":");
-                            PlayerProfile temporaryProfile = loadFromLine(rawSplitData);
-                            writeUserToLine(temporaryProfile, rawSplitData[USERNAME_INDEX], temporaryProfile.getUniqueId(), writer);
-                        } else {
-                            writer.append(line).append("\r\n");
-                        }
-
-                    }
-
-                    // Write the new file
-                    out = new FileWriter(usersFilePath);
-                    out.write(writer.toString());
-                }
-                catch (IOException e) {
-                    logger.severe("Exception while reading " + usersFilePath + " (Are you sure you formatted it correctly?)" + e);
-                }
-                finally {
-                    if (in != null) {
-                        try {
-                            in.close();
-                        }
-                        catch (IOException e) {
-                            e.printStackTrace();
-                        }
-                    }
-                    if (out != null) {
-                        try {
-                            out.close();
-                        }
-                        catch (IOException e) {
-                            e.printStackTrace();
-                        }
-                    }
-                }
             }
-
-            if(corruptDataFound)
-                logger.info("Corrupt data was found and removed, everything should be working fine. It is possible some player data was lost.");
         }
 
-        usersFile.getParentFile().mkdir();
+        if (fileWriter != null) {
-
+            try {
-        try {
+                fileWriter.close();
-            logger.info("Creating mcmmo.users file...");
+            }
-            new File(usersFilePath).createNewFile();
+            catch (IOException e) {
-        }
+                e.printStackTrace();
-        catch (IOException e) {
+            }
-            e.printStackTrace();
-        }
-
-        if(corruptDataFound) {
-            return 1;
-        } else if(oldDataFound) {
-            return 2;
-        } else {
-            return 0;
         }
     }
 

src/main/java/com/gmail/nossr50/database/flatfile/BadCategorizedFlatFileData.java

@@ -0,0 +1,42 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import com.google.common.base.Objects;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Arrays;
+import java.util.HashSet;
+
+public class BadCategorizedFlatFileData extends CategorizedFlatFileData {
+    private final boolean[] badDataIndexes;
+
+    protected BadCategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String[] splitData, boolean[] badDataIndexes) {
+        super(uniqueProcessingId, dataFlags, splitData);
+        this.badDataIndexes = badDataIndexes;
+    }
+
+    public boolean[] getBadDataIndexes() {
+        return badDataIndexes;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        if (!super.equals(o)) return false;
+        BadCategorizedFlatFileData that = (BadCategorizedFlatFileData) o;
+        return Objects.equal(badDataIndexes, that.badDataIndexes);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(super.hashCode(), badDataIndexes);
+    }
+
+    @Override
+    public String toString() {
+        return "BadCategorizedFlatFileData{" +
+                "badDataIndexes=" + Arrays.toString(badDataIndexes) +
+                '}';
+    }
+}

src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileData.java

@@ -1,8 +1,7 @@
 package com.gmail.nossr50.database.flatfile;
 
-import com.gmail.nossr50.database.FlatFileDataContainer;
 import com.gmail.nossr50.database.FlatFileDataFlag;
-import com.gmail.nossr50.database.FlatFileDatabaseManager;
+import com.google.common.base.Objects;
 import org.jetbrains.annotations.NotNull;
 
 import java.util.HashSet;
@@ -10,30 +9,21 @@ import java.util.Set;
 
 public class CategorizedFlatFileData implements FlatFileDataContainer {
     private final @NotNull Set<FlatFileDataFlag> dataFlags;
-    private final @NotNull String stringDataRepresentation;
+    private final @NotNull String[] splitData;
     private final int uniqueProcessingId;
-    private final boolean[] badDataIndexes;
 
-    protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation) {
+    protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String[] splitData) {
         this.uniqueProcessingId = uniqueProcessingId;
         this.dataFlags = dataFlags;
-        this.stringDataRepresentation = stringDataRepresentation;
+        this.splitData = splitData;
-        badDataIndexes = new boolean[FlatFileDatabaseManager.DATA_ENTRY_COUNT];
-    }
-
-    protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation, boolean[] badDataIndexes) {
-        this.uniqueProcessingId = uniqueProcessingId;
-        this.dataFlags = dataFlags;
-        this.stringDataRepresentation = stringDataRepresentation;
-        this.badDataIndexes = badDataIndexes;
     }
 
     public @NotNull Set<FlatFileDataFlag> getDataFlags() {
         return dataFlags;
     }
 
-    public @NotNull String getStringDataRepresentation() {
+    public @NotNull String[] getSplitData() {
-        return stringDataRepresentation;
+        return splitData;
     }
 
     public int getUniqueProcessingId() {
@@ -44,7 +34,25 @@ public class CategorizedFlatFileData implements FlatFileDataContainer {
         return dataFlags.size() == 0;
     }
 
-    public boolean[] getBadDataIndexes() {
+    @Override
-        return badDataIndexes;
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        CategorizedFlatFileData that = (CategorizedFlatFileData) o;
+        return uniqueProcessingId == that.uniqueProcessingId && Objects.equal(dataFlags, that.dataFlags) && Objects.equal(splitData, that.splitData);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(dataFlags, splitData, uniqueProcessingId);
+    }
+
+    @Override
+    public String toString() {
+        return "CategorizedFlatFileData{" +
+                "dataFlags=" + dataFlags +
+                ", stringDataRepresentation='" + splitData + '\'' +
+                ", uniqueProcessingId=" + uniqueProcessingId +
+                '}';
     }
 }

src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileDataBuilder.java

@@ -1,32 +0,0 @@
-package com.gmail.nossr50.database.flatfile;
-
-import com.gmail.nossr50.database.FlatFileDataFlag;
-import org.jetbrains.annotations.NotNull;
-
-import java.util.HashSet;
-
-public class CategorizedFlatFileDataBuilder {
-    private final @NotNull HashSet<FlatFileDataFlag> dataFlags;
-    private @NotNull String stringDataRepresentation;
-    private final int uniqueProcessingId;
-
-    public CategorizedFlatFileDataBuilder(@NotNull String stringDataRepresentation, int uniqueProcessingId) {
-        this.uniqueProcessingId = uniqueProcessingId;
-        this.stringDataRepresentation = stringDataRepresentation;
-        dataFlags = new HashSet<>();
-    }
-
-    public CategorizedFlatFileDataBuilder appendFlag(@NotNull FlatFileDataFlag dataFlag) {
-        dataFlags.add(dataFlag);
-        return this;
-    }
-
-    public CategorizedFlatFileData build() {
-        return new CategorizedFlatFileData(uniqueProcessingId, dataFlags, stringDataRepresentation);
-    }
-
-    public CategorizedFlatFileDataBuilder setStringDataRepresentation(@NotNull String stringDataRepresentation) {
-        this.stringDataRepresentation = stringDataRepresentation;
-        return this;
-    }
-}

src/main/java/com/gmail/nossr50/database/flatfile/FlatFileDataBuilder.java

@@ -0,0 +1,42 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.HashSet;
+
+public class FlatFileDataBuilder {
+    private final @NotNull HashSet<FlatFileDataFlag> dataFlags;
+    private @NotNull String[] splitStringData;
+    private final int uniqueProcessingId;
+    private boolean[] badDataValues;
+
+    public FlatFileDataBuilder(@NotNull String[] splitStringData, int uniqueProcessingId) {
+        this.uniqueProcessingId = uniqueProcessingId;
+        this.splitStringData = splitStringData;
+        dataFlags = new HashSet<>();
+    }
+
+    public @NotNull FlatFileDataBuilder appendFlag(@NotNull FlatFileDataFlag dataFlag) {
+        dataFlags.add(dataFlag);
+        return this;
+    }
+
+    public @NotNull FlatFileDataBuilder appendBadDataValues(boolean[] badDataValues) {
+        this.badDataValues = badDataValues;
+        return this;
+    }
+
+    public @NotNull FlatFileDataContainer build() {
+        if(dataFlags.contains(FlatFileDataFlag.BAD_VALUES)) {
+            return new BadCategorizedFlatFileData(uniqueProcessingId, dataFlags, splitStringData, badDataValues);
+        }
+
+        return new CategorizedFlatFileData(uniqueProcessingId, dataFlags, splitStringData);
+    }
+
+    public @NotNull FlatFileDataBuilder setSplitStringData(@NotNull String[] splitStringData) {
+        this.splitStringData = splitStringData;
+        return this;
+    }
+}

src/main/java/com/gmail/nossr50/database/flatfile/FlatFileDataContainer.java

@@ -0,0 +1,21 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Set;
+
+public interface FlatFileDataContainer {
+    default @Nullable Set<FlatFileDataFlag> getDataFlags() {
+        return null;
+    }
+
+    @NotNull String[] getSplitData();
+
+    int getUniqueProcessingId();
+
+    default boolean isHealthyData() {
+        return getDataFlags() == null || getDataFlags().size() == 0;
+    }
+}

src/main/java/com/gmail/nossr50/database/flatfile/FlatFileSaveDataProcessor.java

@@ -0,0 +1,117 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import com.gmail.nossr50.database.FlatFileDatabaseManager;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import static com.gmail.nossr50.database.FlatFileDatabaseManager.*;
+import static com.gmail.nossr50.database.FlatFileDatabaseManager.UUID_INDEX;
+
+public class FlatFileSaveDataProcessor {
+
+    public static @Nullable String[] getPreparedSaveDataLine(@NotNull FlatFileDataContainer dataContainer) {
+        if(dataContainer.getDataFlags() == null) {
+            return dataContainer.getSplitData();
+        }
+
+        //Data of this type is not salvageable
+        //TODO: Test that we ignore the things we are supposed to ignore
+        //TODO: Should we even keep track of the bad data or just not even build data containers for it? Making containers for it is only really useful for debugging.. well I suppose operations are typically async so it shouldn't matter
+        if(dataContainer.getDataFlags().contains(FlatFileDataFlag.JUNK)
+                || dataContainer.getDataFlags().contains(FlatFileDataFlag.DUPLICATE_UUID) //For now we will not try to fix any issues with UUIDs
+                || dataContainer.getDataFlags().contains(FlatFileDataFlag.BAD_UUID_DATA) //For now we will not try to fix any issues with UUIDs
+                || dataContainer.getDataFlags().contains(FlatFileDataFlag.TOO_INCOMPLETE)
+                || dataContainer.getDataFlags().contains(FlatFileDataFlag.EMPTY_LINE)) {
+            return null;
+        }
+
+        String[] splitData;
+
+        /*
+         * First fix the bad data values if they exist
+         */
+        if(dataContainer instanceof BadCategorizedFlatFileData) {
+            BadCategorizedFlatFileData badData = (BadCategorizedFlatFileData) dataContainer;
+            splitData = repairBadData(dataContainer.getSplitData(), badData.getBadDataIndexes());
+        } else {
+            splitData = dataContainer.getSplitData();
+        }
+
+        //Make sure we have as many values as we are supposed to
+        assert splitData.length == FlatFileDatabaseManager.DATA_ENTRY_COUNT;
+        return splitData;
+    }
+
+    public static @NotNull String[] repairBadData(@NotNull String[] splitData, boolean[] badDataValues) {
+        for(int i = 0; i < FlatFileDatabaseManager.DATA_ENTRY_COUNT; i++) {
+            if(badDataValues[i]) {
+                //This data value was marked as bad so we zero initialize it
+                splitData[i] = getZeroInitialisedData(i, 0);
+            }
+        }
+
+        return splitData;
+    }
+
+    /**
+     * @param index "zero" Initialization will depend on what the index is for
+     * @return the "zero" initialized data corresponding to the index
+     */
+    public static @NotNull String getZeroInitialisedData(int index, int startingLevel) throws IndexOutOfBoundsException {
+        switch(index) {
+            case USERNAME_INDEX:
+                return LEGACY_INVALID_OLD_USERNAME; //We'll keep using this value for legacy compatibility reasons (not sure if needed but don't care)
+            case 2: //Assumption: Used to be for something, no longer used
+            case 3: //Assumption: Used to be for something, no longer used
+            case 23: //Assumption: Used to be used for something, no longer used
+            case 33: //Assumption: Used to be used for something, no longer used
+            case HEALTHBAR:
+                return "IGNORED";
+            case SKILLS_MINING:
+            case SKILLS_REPAIR:
+            case SKILLS_UNARMED:
+            case SKILLS_HERBALISM:
+            case SKILLS_EXCAVATION:
+            case SKILLS_ARCHERY:
+            case SKILLS_SWORDS:
+            case SKILLS_AXES:
+            case SKILLS_WOODCUTTING:
+            case SKILLS_ACROBATICS:
+            case SKILLS_TAMING:
+            case SKILLS_FISHING:
+            case SKILLS_ALCHEMY:
+                return String.valueOf(startingLevel);
+            case LAST_LOGIN:
+                return String.valueOf(System.currentTimeMillis() / 1000); //This is just to shorten the value
+            case COOLDOWN_BERSERK:
+            case COOLDOWN_GIGA_DRILL_BREAKER:
+            case COOLDOWN_TREE_FELLER:
+            case COOLDOWN_GREEN_TERRA:
+            case COOLDOWN_SERRATED_STRIKES:
+            case COOLDOWN_SKULL_SPLITTER:
+            case COOLDOWN_SUPER_BREAKER:
+            case COOLDOWN_BLAST_MINING:
+            case SCOREBOARD_TIPS:
+            case COOLDOWN_CHIMAERA_WING:
+            case EXP_MINING:
+            case EXP_WOODCUTTING:
+            case EXP_REPAIR:
+            case EXP_UNARMED:
+            case EXP_HERBALISM:
+            case EXP_EXCAVATION:
+            case EXP_ARCHERY:
+            case EXP_SWORDS:
+            case EXP_AXES:
+            case EXP_ACROBATICS:
+            case EXP_TAMING:
+            case EXP_FISHING:
+            case EXP_ALCHEMY:
+                return "0";
+            case UUID_INDEX:
+                throw new IndexOutOfBoundsException(); //TODO: Add UUID recovery? Might not even be worth it.
+        }
+
+        throw new IndexOutOfBoundsException();
+    }
+}

src/test/java/com/gmail/nossr50/database/FlatFileDatabaseManagerTest.java

@@ -48,28 +48,34 @@ public class FlatFileDatabaseManagerTest {
             "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
     };
 
+    private static final String[] badUUIDDatabaseData = {
+            "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
+            "z750:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:3:5:1600906906:", //This one has an incorrect UUID representation
+            "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
+    };
+
     private static final String[] outdatedDatabaseData = {
             "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
             "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
-            "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:" //This user is missing data added after UUID index
+            "electronicboy:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:" //This user is missing data added after UUID index
     };
 
     private static final String[] emptyLineDatabaseData = {
             "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
             "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
-            "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:",
+            "kashike:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:",
             "" //EMPTY LINE
     };
 
     private static final String[] emptyNameDatabaseData = {
             ":1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
             "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
-            "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
+            "aikar:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
     };
 
     private static final String[] duplicateNameDatabaseData = {
-            "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
+            "mochi:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
-            "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:631e3896-da2a-4077-974b-d047859d76bc:0:0:",
+            "mochi:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:631e3896-da2a-4077-974b-d047859d76bc:0:0:",
     };
 
     private static final String[] duplicateUUIDDatabaseData = {
@@ -109,38 +115,43 @@ public class FlatFileDatabaseManagerTest {
     }
 
     @Test
-    public void testFindDuplicateNames() {
+    public void testFindFixableDuplicateNames() {
-        addDataAndCheckForFlag(db, duplicateNameDatabaseData, FlatFileDataFlag.DUPLICATE_NAME_FIXABLE);
+        overwriteDataAndCheckForFlag(db, duplicateNameDatabaseData, FlatFileDataFlag.DUPLICATE_NAME);
     }
 
     @Test
     public void testFindDuplicateUUIDs() {
-        addDataAndCheckForFlag(db, duplicateUUIDDatabaseData, FlatFileDataFlag.DUPLICATE_UUID);
+        overwriteDataAndCheckForFlag(db, duplicateUUIDDatabaseData, FlatFileDataFlag.DUPLICATE_UUID);
+    }
+
+    @Test()
+    public void findBadUUIDData() {
+        overwriteDataAndCheckForFlag(db, badUUIDDatabaseData, FlatFileDataFlag.BAD_UUID_DATA);
     }
 
     @Test
     public void testFindCorruptData() {
-        addDataAndCheckForFlag(db, corruptDatabaseData, FlatFileDataFlag.JUNK);
+        overwriteDataAndCheckForFlag(db, corruptDatabaseData, FlatFileDataFlag.JUNK);
     }
 
     @Test
     public void testFindEmptyNames() {
-        addDataAndCheckForFlag(db, emptyNameDatabaseData, FlatFileDataFlag.MISSING_NAME);
+        overwriteDataAndCheckForFlag(db, emptyNameDatabaseData, FlatFileDataFlag.MISSING_NAME);
     }
 
-    @Test
+//    @Test
-    public void testFindEmptyLine() {
+//    public void testFindEmptyLine() {
-        addDataAndCheckForFlag(db, emptyLineDatabaseData, FlatFileDataFlag.EMPTY_LINE);
+//        overwriteDataAndCheckForFlag(db, emptyLineDatabaseData, FlatFileDataFlag.EMPTY_LINE);
-    }
+//    }
 
     @Test
     public void testFindBadValues() {
-        addDataAndCheckForFlag(db, badDatabaseData, FlatFileDataFlag.BAD_VALUES);
+        overwriteDataAndCheckForFlag(db, badDatabaseData, FlatFileDataFlag.BAD_VALUES);
     }
 
     @Test
     public void testFindOutdatedData() {
-        addDataAndCheckForFlag(db, outdatedDatabaseData, FlatFileDataFlag.INCOMPLETE);
+        overwriteDataAndCheckForFlag(db, outdatedDatabaseData, FlatFileDataFlag.INCOMPLETE);
     }
 
     @Test
@@ -149,7 +160,6 @@ public class FlatFileDatabaseManagerTest {
         assertEquals(db.getDatabaseType(), DatabaseType.FLATFILE);
     }
 
-
     private void replaceDataInFile(@NotNull FlatFileDatabaseManager flatFileDatabaseManager, @NotNull String[] dataEntries) {
         String filePath = flatFileDatabaseManager.getUsersFile().getAbsolutePath();
         BufferedReader in = null;
@@ -203,7 +213,7 @@ public class FlatFileDatabaseManagerTest {
 
     }
 
-    private void addDataAndCheckForFlag(@NotNull FlatFileDatabaseManager targetDatabase, @NotNull String[] data, @NotNull FlatFileDataFlag flag) {
+    private void overwriteDataAndCheckForFlag(@NotNull FlatFileDatabaseManager targetDatabase, @NotNull String[] data, @NotNull FlatFileDataFlag flag) {
         replaceDataInFile(targetDatabase, data);
 
         List<FlatFileDataFlag> dataFlags = targetDatabase.checkFileHealthAndStructure();