FlatFileDatabaseManager refactor + adding tests part 1

Changelog.txt

@@ -1,4 +1,7 @@
 Version 2.1.189
+    Rewrote how FlatFileDatabase verifies data integrity
+    Added unit tests for FlatFileDatabaseManager (see notes)
+    Fixed a bug where FlatFileDatabaseManager didn't properly upgrade older database entries to the newest schema
     The setting to disable the mcMMO user block tracker has been moved from our "hidden config" to persistent_data.yml
     Added 'mcMMO_Region_System.Enabled' to persistent_data.yml (don't touch this setting unless you know what you are doing)
     Fixed a bug that would remove components from death messages when players were killed by mobs (thanks lexikiq)
@@ -12,9 +15,9 @@ Version 2.1.189
     (API) PrimarySkillType will soon be just an enum with nothing special going on
     (API) Deprecated the members of PrimarySkillType use mcMMO::getSkillTools instead, deprecated members will be removed in Tridents & Crossbows (due soon)
     (API) Some members of PrimarySkillType were removed and not deprecated (such as the field constants)
-    Added unit tests for FlatFileDatabaseManager
 
     NOTES:
+    The tests added for FlatFileDatabase will help make sure bugs don't result in any loss of data
     Ultra Permissions is SAFE to use with mcMMO
     After getting in contact with the UltraPermissions devs and exhaustive testing, I have concluded that using UltraPermissions is completely safe with mcMMO. The users who had an issue with performance currently have an unknown cause, potentially it is from a plugin using the UltraPermissions API I really can't say without more data. My apologies to the UltraPermissions team for reporting an issue between our two plugins directly, as that is not the case. I would have tested it myself sooner but UltraPermissions was closed source and premium so I wasn't particularly motivated to do so, however I have been given access to the binaries so now I can do all the testing I want if future issues ever arise which I have zero expectations that they will.
 

src/main/java/com/gmail/nossr50/commands/database/ConvertDatabaseCommand.java

@@ -33,8 +33,6 @@ public class ConvertDatabaseCommand implements CommandExecutor {
                 return true;
             }
 
-            oldDatabase.init();
-
             if (previousType == DatabaseType.CUSTOM) {
                 Class<?> clazz;
 
@@ -47,7 +45,6 @@ public class ConvertDatabaseCommand implements CommandExecutor {
                     }
 
                     oldDatabase = DatabaseManagerFactory.createCustomDatabaseManager((Class<? extends DatabaseManager>) clazz);
-                    oldDatabase.init();
                 } catch (Throwable e) {
                     e.printStackTrace();
                     sender.sendMessage(LocaleLoader.getString("Commands.mcconvert.Database.InvalidType", args[1]));

src/main/java/com/gmail/nossr50/database/DatabaseManager.java

@@ -73,8 +73,6 @@ public interface DatabaseManager {
      */
     Map<PrimarySkillType, Integer> readRank(String playerName);
 
-    default void init() {};
-
     /**
      * Add a new user to the database.
      *

src/main/java/com/gmail/nossr50/database/ExpectedType.java

@@ -0,0 +1,12 @@
+package com.gmail.nossr50.database;
+
+public enum ExpectedType {
+    STRING,
+    INTEGER,
+    BOOLEAN,
+    FLOAT,
+    DOUBLE,
+    UUID,
+    IGNORED,
+    OUT_OF_RANGE
+}

src/main/java/com/gmail/nossr50/database/FlatFileDataContainer.java

@@ -0,0 +1,5 @@
+package com.gmail.nossr50.database;
+
+//Marker interface
+public interface FlatFileDataContainer {
+}

src/main/java/com/gmail/nossr50/database/FlatFileDataFlag.java

@@ -0,0 +1,14 @@
+package com.gmail.nossr50.database;
+
+public enum FlatFileDataFlag {
+    INCOMPLETE,
+    BAD_VALUES,
+    MISSING_NAME,
+    DUPLICATE_NAME_FIXABLE,
+    DUPLICATE_NAME_NOT_FIXABLE,
+    DUPLICATE_UUID,
+    MISSING_OR_NULL_UUID,
+    TOO_INCOMPLETE,
+    JUNK,
+    EMPTY,
+}

src/main/java/com/gmail/nossr50/database/FlatFileDataProcessor.java

@@ -0,0 +1,303 @@
+package com.gmail.nossr50.database;
+
+import com.gmail.nossr50.database.flatfile.CategorizedFlatFileData;
+import com.gmail.nossr50.database.flatfile.CategorizedFlatFileDataBuilder;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.File;
+import java.util.*;
+import java.util.logging.Logger;
+
+import static com.gmail.nossr50.database.FlatFileDatabaseManager.*;
+
+public class FlatFileDataProcessor {
+    public static final String INVALID_OLD_USERNAME = "_INVALID_OLD_USERNAME_";
+    private @NotNull List<CategorizedFlatFileData> categorizedDataList;
+    private @NotNull List<FlatFileDataFlag> flatFileDataFlags;
+    private final @NotNull File userFile;
+    private final @NotNull Logger logger;
+    private final HashSet<String> names;
+    private final HashSet<UUID> uuids;
+    private int uniqueProcessingID;
+    boolean corruptDataFound;
+
+    public FlatFileDataProcessor(@NotNull File userFile, @NotNull Logger logger) {
+        this.userFile = userFile;
+        this.logger = logger;
+        categorizedDataList = new ArrayList<>();
+        flatFileDataFlags = new ArrayList<>();
+        names = new HashSet<>();
+        uuids = new HashSet<>();
+        uniqueProcessingID = 0;
+    }
+
+    public void processData(@NotNull String lineData) {
+        CategorizedFlatFileDataBuilder builder = new CategorizedFlatFileDataBuilder(lineData, uniqueProcessingID);
+        uniqueProcessingID++;
+
+        /*
+         * Is the line empty?
+         */
+        if (lineData.isEmpty()) {
+            registerData(builder.appendFlag(FlatFileDataFlag.EMPTY));
+            return;
+        }
+
+        //Make sure the data line is "correct"
+        if(lineData.charAt(lineData.length() - 1) != ':') {
+            // Length checks depend on last rawSplitData being ':'
+            // We add it here if it is missing
+            lineData = lineData.concat(":");
+        }
+
+        //Split the data into an array
+        String[] splitDataLine = lineData.split(":");
+
+        //This is the minimum size of the split array needed to be considered proper data
+        if(splitDataLine.length < getMinimumSplitDataLength()) {
+            //Data is considered junk
+            if(!corruptDataFound) {
+                logger.severe("Some corrupt data was found in mcmmo.users and has been repaired, it is possible that some player data has been lost in this process.");
+                corruptDataFound = true;
+            }
+
+            if(splitDataLine.length >= 10 //The value here is kind of arbitrary, it shouldn't be too low to avoid false positives, but also we aren't really going to correctly identify when player data has been corrupted or not with 100% accuracy ever
+                    && splitDataLine[0] != null && !splitDataLine[0].isEmpty()) {
+                if(splitDataLine[0].length() <= 16 && splitDataLine[0].length() >= 3) {
+                    logger.severe("Not enough data found to recover corrupted player data for user: "+splitDataLine[0]);
+                    registerData(builder.appendFlag(FlatFileDataFlag.TOO_INCOMPLETE));
+                    return;
+                }
+            } else {
+                registerData(builder.appendFlag(FlatFileDataFlag.JUNK));
+                return;
+            }
+        }
+
+        /*
+         * Check for duplicate names
+         */
+
+        boolean nameIsDupe = false;
+        boolean invalidUUID = false;
+
+        String name = splitDataLine[USERNAME_INDEX];
+        String strOfUUID = splitDataLine[UUID_INDEX];
+
+        if(name.isEmpty()) {
+            reportBadDataLine("No name found for data", "[MISSING NAME]", lineData);
+            builder.appendFlag(FlatFileDataFlag.MISSING_NAME);
+        }
+
+        if(strOfUUID.isEmpty() || strOfUUID.equalsIgnoreCase("NULL")) {
+            invalidUUID = true;
+            reportBadDataLine("Empty/null UUID for user", "Empty/null", lineData);
+            builder.appendFlag(FlatFileDataFlag.MISSING_OR_NULL_UUID);
+        }
+
+        UUID uuid = null;
+
+        try {
+            uuid = UUID.fromString(strOfUUID);
+        } catch (IllegalArgumentException e) {
+            invalidUUID = true;
+            //UUID does not conform
+
+            reportBadDataLine("Invalid UUID data found for user", strOfUUID, lineData);
+            e.printStackTrace();
+        }
+
+        //Duplicate UUID is no good, reject them
+        if(uuid != null && uuids.contains(uuid)) {
+            registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_UUID));
+            return;
+        }
+
+        if(names.contains(name)) {
+            //Duplicate entry
+            nameIsDupe = true;
+
+            //We can accept them if they are a duped name if they have a unique UUID
+            if(invalidUUID) {
+                //Reject the data
+                reportBadDataLine("Duplicate user found and due to a missing UUID their data had to be discarded", name, lineData);
+
+                registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_NOT_FIXABLE));
+                return;
+            } else {
+                builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_FIXABLE);
+            }
+        }
+
+        //Make sure the data is up to date schema wise
+        if(splitDataLine.length < DATA_ENTRY_COUNT) {
+            String[] correctSizeSplitData = Arrays.copyOf(splitDataLine, DATA_ENTRY_COUNT);
+            lineData = org.apache.commons.lang.StringUtils.join(correctSizeSplitData, ":") + ":";
+            splitDataLine = lineData.split(":");
+            builder.appendFlag(FlatFileDataFlag.INCOMPLETE);
+            builder.setStringDataRepresentation(lineData);
+        }
+
+        /*
+         * After establishing this data has at least an identity we check for bad data
+         * Bad Value checks
+         */
+
+        //Check each data for bad values
+        boolean[] badDataValues = new boolean[DATA_ENTRY_COUNT];
+        boolean anyBadData = false;
+
+        for(int i = 0; i < DATA_ENTRY_COUNT; i++) {
+            if(shouldNotBeEmpty(splitDataLine[i], i)) {
+                badDataValues[i] = true;
+                anyBadData = true;
+                reportBadDataLine("Data is empty when it should not be at index", "[EMPTY]", lineData);
+                continue;
+            }
+
+            boolean isCorrectType = isOfExpectedType(splitDataLine[i], getExpectedValueType(i));
+
+            if(!isCorrectType) {
+                reportBadDataLine("Data is not of correct type", splitDataLine[i], lineData);
+                anyBadData = true;
+                badDataValues[i] = true;
+            }
+        }
+
+        if(anyBadData) {
+            builder.appendFlag(FlatFileDataFlag.BAD_VALUES);
+        }
+    }
+
+    public boolean shouldNotBeEmpty(String data, int index) {
+        if(getExpectedValueType(index) == ExpectedType.IGNORED) {
+            return false;
+        } else {
+            return data.isEmpty();
+        }
+    }
+
+    public boolean isOfExpectedType(@NotNull String data, @NotNull ExpectedType expectedType) {
+        switch(expectedType) {
+            case STRING:
+                return true;
+            case INTEGER:
+                try {
+                    Integer.valueOf(data);
+                    return true;
+                } catch (Exception e) {
+                    return false;
+                }
+            case BOOLEAN:
+                return data.equalsIgnoreCase("true") || data.equalsIgnoreCase("false");
+            case FLOAT:
+                try {
+                    Float.valueOf(data);
+                    return true;
+                } catch (NumberFormatException e) {
+                    return false;
+                }
+            case DOUBLE:
+                try {
+                    Double.valueOf(data);
+                    return true;
+                } catch (NumberFormatException e) {
+                    return false;
+                }
+            case UUID:
+                try {
+                    UUID.fromString(data);
+                    return true;
+                } catch (IllegalArgumentException e) {
+                    return false;
+                }
+            case OUT_OF_RANGE:
+                throw new ArrayIndexOutOfBoundsException("Value matched type OUT_OF_RANGE, this should never happen.");
+            case IGNORED:
+            default:
+                return true;
+        }
+
+    }
+
+    private void reportBadDataLine(String warning, String context, String dataLine) {
+        logger.severe("FlatFileDatabaseBuilder Warning: " + warning + " - " + context);
+        logger.severe("FlatFileDatabaseBuilder: (Line Data) - " + dataLine);
+    }
+
+    private int getMinimumSplitDataLength() {
+        return UUID_INDEX + 1;
+    }
+
+    private void registerData(@NotNull CategorizedFlatFileDataBuilder builder) {
+        CategorizedFlatFileData categorizedFlatFileData = builder.build();
+        categorizedDataList.add(categorizedFlatFileData);
+        flatFileDataFlags.addAll(categorizedFlatFileData.getDataFlags());
+    }
+
+    public @NotNull ExpectedType getExpectedValueType(int dataIndex) {
+        switch(dataIndex) {
+            case USERNAME_INDEX:
+                return ExpectedType.STRING;
+            case 2: //Used to be for something, no longer used
+            case 3: //Used to be for something, no longer used
+            case HEALTHBAR:
+                return ExpectedType.IGNORED;
+            case SKILLS_MINING:
+            case SKILLS_REPAIR:
+            case SKILLS_UNARMED:
+            case SKILLS_HERBALISM:
+            case SKILLS_EXCAVATION:
+            case SKILLS_ARCHERY:
+            case SKILLS_SWORDS:
+            case SKILLS_AXES:
+            case SKILLS_WOODCUTTING:
+            case SKILLS_ACROBATICS:
+            case SKILLS_TAMING:
+            case SKILLS_FISHING:
+            case SKILLS_ALCHEMY:
+            case LAST_LOGIN:
+            case COOLDOWN_BERSERK:
+            case COOLDOWN_GIGA_DRILL_BREAKER:
+            case COOLDOWN_TREE_FELLER:
+            case COOLDOWN_GREEN_TERRA:
+            case COOLDOWN_SERRATED_STRIKES:
+            case COOLDOWN_SKULL_SPLITTER:
+            case COOLDOWN_SUPER_BREAKER:
+            case COOLDOWN_BLAST_MINING:
+            case SCOREBOARD_TIPS:
+            case COOLDOWN_CHIMAERA_WING:
+                return ExpectedType.INTEGER;
+            case EXP_MINING:
+            case EXP_WOODCUTTING:
+            case EXP_REPAIR:
+            case EXP_UNARMED:
+            case EXP_HERBALISM:
+            case EXP_EXCAVATION:
+            case EXP_ARCHERY:
+            case EXP_SWORDS:
+            case EXP_AXES:
+            case EXP_ACROBATICS:
+            case EXP_TAMING:
+            case EXP_FISHING:
+            case EXP_ALCHEMY:
+                return ExpectedType.FLOAT;
+            case UUID_INDEX:
+                return ExpectedType.UUID;
+            default:
+                return ExpectedType.OUT_OF_RANGE;
+        }
+    }
+
+    public @NotNull List<CategorizedFlatFileData> getCategorizedDataList() {
+        return categorizedDataList;
+    }
+
+    public @NotNull List<FlatFileDataFlag> getFlatFileDataFlags() {
+        return flatFileDataFlags;
+    }
+
+    public int getDataFlagCount() {
+        return flatFileDataFlags.size();
+    }
+}

src/main/java/com/gmail/nossr50/database/FlatFileDatabaseManager.java

@@ -32,48 +32,48 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
     private final @NotNull File usersFile;
     private static final Object fileWritingLock = new Object();
 
-    public static int USERNAME_INDEX = 0;
-    public static int SKILLS_MINING = 1;
-    public static int EXP_MINING = 4;
-    public static int SKILLS_WOODCUTTING = 5;
-    public static int EXP_WOODCUTTING = 6;
-    public static int SKILLS_REPAIR = 7;
-    public static int SKILLS_UNARMED = 8;
-    public static int SKILLS_HERBALISM = 9;
-    public static int SKILLS_EXCAVATION = 10;
-    public static int SKILLS_ARCHERY = 11;
-    public static int SKILLS_SWORDS = 12;
-    public static int SKILLS_AXES = 13;
-    public static int SKILLS_ACROBATICS = 14;
-    public static int EXP_REPAIR = 15;
-    public static int EXP_UNARMED = 16;
-    public static int EXP_HERBALISM = 17;
-    public static int EXP_EXCAVATION = 18;
-    public static int EXP_ARCHERY = 19;
-    public static int EXP_SWORDS = 20;
-    public static int EXP_AXES = 21;
-    public static int EXP_ACROBATICS = 22;
-    public static int SKILLS_TAMING = 24;
-    public static int EXP_TAMING = 25;
-    public static int COOLDOWN_BERSERK = 26;
-    public static int COOLDOWN_GIGA_DRILL_BREAKER = 27;
-    public static int COOLDOWN_TREE_FELLER = 28;
-    public static int COOLDOWN_GREEN_TERRA = 29;
-    public static int COOLDOWN_SERRATED_STRIKES = 30;
-    public static int COOLDOWN_SKULL_SPLITTER = 31;
-    public static int COOLDOWN_SUPER_BREAKER = 32;
-    public static int SKILLS_FISHING = 34;
-    public static int EXP_FISHING = 35;
-    public static int COOLDOWN_BLAST_MINING = 36;
-    public static int LAST_LOGIN = 37;
-    public static int HEALTHBAR = 38;
-    public static int SKILLS_ALCHEMY = 39;
-    public static int EXP_ALCHEMY = 40;
-    public static int UUID_INDEX = 41;
-    public static int SCOREBOARD_TIPS = 42;
-    public static int COOLDOWN_CHIMAERA_WING = 43;
+    public static final int USERNAME_INDEX = 0;
+    public static final int SKILLS_MINING = 1;
+    public static final int EXP_MINING = 4;
+    public static final int SKILLS_WOODCUTTING = 5;
+    public static final int EXP_WOODCUTTING = 6;
+    public static final int SKILLS_REPAIR = 7;
+    public static final int SKILLS_UNARMED = 8;
+    public static final int SKILLS_HERBALISM = 9;
+    public static final int SKILLS_EXCAVATION = 10;
+    public static final int SKILLS_ARCHERY = 11;
+    public static final int SKILLS_SWORDS = 12;
+    public static final int SKILLS_AXES = 13;
+    public static final int SKILLS_ACROBATICS = 14;
+    public static final int EXP_REPAIR = 15;
+    public static final int EXP_UNARMED = 16;
+    public static final int EXP_HERBALISM = 17;
+    public static final int EXP_EXCAVATION = 18;
+    public static final int EXP_ARCHERY = 19;
+    public static final int EXP_SWORDS = 20;
+    public static final int EXP_AXES = 21;
+    public static final int EXP_ACROBATICS = 22;
+    public static final int SKILLS_TAMING = 24;
+    public static final int EXP_TAMING = 25;
+    public static final int COOLDOWN_BERSERK = 26;
+    public static final int COOLDOWN_GIGA_DRILL_BREAKER = 27;
+    public static final int COOLDOWN_TREE_FELLER = 28;
+    public static final int COOLDOWN_GREEN_TERRA = 29;
+    public static final int COOLDOWN_SERRATED_STRIKES = 30;
+    public static final int COOLDOWN_SKULL_SPLITTER = 31;
+    public static final int COOLDOWN_SUPER_BREAKER = 32;
+    public static final int SKILLS_FISHING = 34;
+    public static final int EXP_FISHING = 35;
+    public static final int COOLDOWN_BLAST_MINING = 36;
+    public static final int LAST_LOGIN = 37;
+    public static final int HEALTHBAR = 38;
+    public static final int SKILLS_ALCHEMY = 39;
+    public static final int EXP_ALCHEMY = 40;
+    public static final int UUID_INDEX = 41;
+    public static final int SCOREBOARD_TIPS = 42;
+    public static final int COOLDOWN_CHIMAERA_WING = 43;
 
-    public static int DATA_ENTRY_COUNT = COOLDOWN_CHIMAERA_WING + 1; //Update this everytime new data is added
+    public static final int DATA_ENTRY_COUNT = COOLDOWN_CHIMAERA_WING + 1; //Update this everytime new data is added
 
     protected FlatFileDatabaseManager(@NotNull String usersFilePath, @NotNull Logger logger, long purgeTime, int startingLevel) {
         usersFile = new File(usersFilePath);
@@ -81,11 +81,18 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         this.logger = logger;
         this.purgeTime = purgeTime;
         this.startingLevel = startingLevel;
-    }
 
-    public void init() {
-        checkStructure();
-        updateLeaderboards();
+        checkFileHealthAndStructure();
+        List<FlatFileDataFlag> flatFileDataFlags = checkFileHealthAndStructure();
+
+        if(flatFileDataFlags != null) {
+            if(flatFileDataFlags.size() > 0) {
+                logger.info("Detected "+flatFileDataFlags.size() + " data entries which need correction.");
+            }
+        }
+
+        checkFileHealthAndStructure();
+//        updateLeaderboards();
     }
 
     public int purgePowerlessUsers() {
@@ -855,7 +862,7 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
     /**
      * Update the leader boards.
      */
-    private void updateLeaderboards() {
+    public void updateLeaderboards() {
         // Only update FFS leaderboards every 10 minutes.. this puts a lot of strain on the server (depending on the size of the database) and should not be done frequently
         if (System.currentTimeMillis() < lastUpdate + UPDATE_WAIT_TIME) {
             return;
@@ -958,11 +965,45 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         playerStatHash.put(PrimarySkillType.ALCHEMY, alchemy);
     }
 
+    public @Nullable List<FlatFileDataFlag> checkFileHealthAndStructure() {
+        FlatFileDataProcessor dataProcessor = null;
+        int dataFlagCount = 0;
+
+        if (usersFile.exists()) {
+            BufferedReader bufferedReader = null;
+
+            synchronized (fileWritingLock) {
+
+                dataProcessor = new FlatFileDataProcessor(usersFile, logger);
+
+                try {
+                    String currentLine;
+                    bufferedReader = new BufferedReader(new FileReader(usersFilePath));
+                    while ((currentLine = bufferedReader.readLine()) != null) {
+                        dataProcessor.processData(currentLine);
+                    }
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+
+                dataFlagCount = dataProcessor.getDataFlagCount();
+            }
+        }
+
+        if(dataProcessor == null || dataProcessor.getFlatFileDataFlags() == null) {
+            return null;
+        } else {
+            return dataProcessor.getFlatFileDataFlags();
+        }
+    }
+
+
     /**
      * Checks that the file is present and valid
      */
-    private void checkStructure() {
+    public int checkFileHealthAndStructureOld() {
         boolean corruptDataFound = false;
+        boolean oldDataFound = false;
 
         if (usersFile.exists()) {
             BufferedReader in = null;
@@ -1030,6 +1071,7 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
 
                         //Correctly size the data (null entries for missing values)
                         if(line.length() < DATA_ENTRY_COUNT) { //TODO: Test this condition
+                            oldDataFound = true;
                             String[] correctSizeSplitData = Arrays.copyOf(rawSplitData, DATA_ENTRY_COUNT);
                             line = org.apache.commons.lang.StringUtils.join(correctSizeSplitData, ":") + ":";
                             rawSplitData = line.split(":");
@@ -1070,8 +1112,6 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
 
             if(corruptDataFound)
                 logger.info("Corrupt data was found and removed, everything should be working fine. It is possible some player data was lost.");
-
-            return;
         }
 
         usersFile.getParentFile().mkdir();
@@ -1083,6 +1123,14 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         catch (IOException e) {
             e.printStackTrace();
         }
+
+        if(corruptDataFound) {
+            return 1;
+        } else if(oldDataFound) {
+            return 2;
+        } else {
+            return 0;
+        }
     }
 
     private Integer getPlayerRank(String playerName, List<PlayerStat> statsList) {
@@ -1239,7 +1287,7 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
         return DatabaseType.FLATFILE;
     }
 
-    public File getUsersFile() {
+    public @NotNull File getUsersFile() {
         return usersFile;
     }
 

src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java

@@ -113,10 +113,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
         poolProperties.setValidationQuery("SELECT 1");
         poolProperties.setValidationInterval(30000);
         loadPool = new DataSource(poolProperties);
-    }
 
-    @Override
-    public void init() {
         checkStructure();
     }
 

src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileData.java

@@ -0,0 +1,50 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataContainer;
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import com.gmail.nossr50.database.FlatFileDatabaseManager;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.HashSet;
+import java.util.Set;
+
+public class CategorizedFlatFileData implements FlatFileDataContainer {
+    private final @NotNull Set<FlatFileDataFlag> dataFlags;
+    private final @NotNull String stringDataRepresentation;
+    private final int uniqueProcessingId;
+    private final boolean[] badDataIndexes;
+
+    protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation) {
+        this.uniqueProcessingId = uniqueProcessingId;
+        this.dataFlags = dataFlags;
+        this.stringDataRepresentation = stringDataRepresentation;
+        badDataIndexes = new boolean[FlatFileDatabaseManager.DATA_ENTRY_COUNT];
+    }
+
+    protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation, boolean[] badDataIndexes) {
+        this.uniqueProcessingId = uniqueProcessingId;
+        this.dataFlags = dataFlags;
+        this.stringDataRepresentation = stringDataRepresentation;
+        this.badDataIndexes = badDataIndexes;
+    }
+
+    public @NotNull Set<FlatFileDataFlag> getDataFlags() {
+        return dataFlags;
+    }
+
+    public @NotNull String getStringDataRepresentation() {
+        return stringDataRepresentation;
+    }
+
+    public int getUniqueProcessingId() {
+        return uniqueProcessingId;
+    }
+
+    public boolean isHealthyData() {
+        return dataFlags.size() == 0;
+    }
+
+    public boolean[] getBadDataIndexes() {
+        return badDataIndexes;
+    }
+}

src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileDataBuilder.java

@@ -0,0 +1,33 @@
+package com.gmail.nossr50.database.flatfile;
+
+import com.gmail.nossr50.database.FlatFileDataFlag;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.HashSet;
+
+public class CategorizedFlatFileDataBuilder {
+    private final @NotNull HashSet<FlatFileDataFlag> dataFlags;
+    private @NotNull String stringDataRepresentation;
+    private final int uniqueProcessingId;
+
+    public CategorizedFlatFileDataBuilder(@NotNull String stringDataRepresentation, int uniqueProcessingId) {
+        this.uniqueProcessingId = uniqueProcessingId;
+        this.stringDataRepresentation = stringDataRepresentation;
+        dataFlags = new HashSet<>();
+    }
+
+    public CategorizedFlatFileDataBuilder appendFlag(@NotNull FlatFileDataFlag dataFlag) {
+        dataFlags.add(dataFlag);
+        return this;
+    }
+
+    public CategorizedFlatFileData build() {
+        assert dataFlags.size() > 0;
+        return new CategorizedFlatFileData(uniqueProcessingId, dataFlags, stringDataRepresentation);
+    }
+
+    public CategorizedFlatFileDataBuilder setStringDataRepresentation(@NotNull String stringDataRepresentation) {
+        this.stringDataRepresentation = stringDataRepresentation;
+        return this;
+    }
+}

src/main/java/com/gmail/nossr50/datatypes/skills/PrimarySkillType.java

@@ -27,6 +27,7 @@ public enum PrimarySkillType {
     TAMING,
     UNARMED,
     WOODCUTTING;
+//    boolean issueWarning = true;
 
     /*
      *  Everything below here will be removed in 2.2 (Tridents & Crossbows)
@@ -47,6 +48,20 @@ public enum PrimarySkillType {
      *  Everything below here will be removed in 2.2 (Tridents & Crossbows)
      */
 
+
+//    private void processWarning() {
+//        if(issueWarning) {
+//            StackTraceElement[] stackTraceElements = Thread.currentThread().getStackTrace();
+//            Bukkit.getScheduler().scheduleSyncDelayedTask(mcMMO.p, () -> {
+//                mcMMO.p.getLogger().severe("A plugin that hooks into mcMMO via the mcMMO API is using soon to be deprecated API calls. Contact the plugin author and inform them to update their code before it breaks.");
+//                mcMMO.p.getLogger().severe("Deprecation Call from: " + stackTraceElements[2].toString());
+//                mcMMO.p.getLogger().severe("This warning will not repeat itself. Nothing is broken for now, but in the future it will be.");
+//            });
+//
+//            issueWarning = !issueWarning;
+//        }
+//    }
+
     /**
      * WARNING: Being removed in an upcoming update, you should be using mcMMO.getSkillTools() instead
      * @return the max level of this skill
@@ -65,7 +80,9 @@ public enum PrimarySkillType {
      * @deprecated this is being removed in an upcoming update, you should be using mcMMO.getSkillTools() instead
      */
     @Deprecated
-    public boolean isSuperAbilityUnlocked(@NotNull Player player) { return mcMMO.p.getSkillTools().isSuperAbilityUnlocked(this, player); }
+    public boolean isSuperAbilityUnlocked(@NotNull Player player) {
+        return mcMMO.p.getSkillTools().isSuperAbilityUnlocked(this, player);
+    }
 
     /**
      * WARNING: Being removed in an upcoming update, you should be using mcMMO.getSkillTools() instead

src/main/java/com/gmail/nossr50/mcMMO.java

@@ -231,7 +231,6 @@ public class mcMMO extends JavaPlugin {
             this.purgeTime = 2630000000L * generalConfig.getOldUsersCutoff();
 
             databaseManager = DatabaseManagerFactory.getDatabaseManager(mcMMO.getUsersFilePath(), getLogger(), purgeTime, mcMMO.p.getAdvancedConfig().getStartingLevel());
-            databaseManager.init();
 
             //Check for the newer API and tell them what to do if its missing
             checkForOutdatedAPI();

src/main/java/com/gmail/nossr50/util/skills/SkillTools.java

@@ -10,6 +10,7 @@ import com.gmail.nossr50.locale.LocaleLoader;
 import com.gmail.nossr50.mcMMO;
 import com.gmail.nossr50.util.Permissions;
 import com.gmail.nossr50.util.text.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
@@ -23,57 +24,70 @@ import java.util.*;
 public class SkillTools {
     private final mcMMO pluginRef;
 
-    //TODO: Should these be hash sets instead of lists?
     //TODO: Figure out which ones we don't need, this was copy pasted from a diff branch
-    public final ImmutableList<String> LOCALIZED_SKILL_NAMES;
-    public final ImmutableList<String> FORMATTED_SUBSKILL_NAMES;
-    public final ImmutableSet<String> EXACT_SUBSKILL_NAMES;
-    public final ImmutableList<PrimarySkillType> CHILD_SKILLS;
-    public final ImmutableList<PrimarySkillType> NON_CHILD_SKILLS;
-    public final ImmutableList<PrimarySkillType> COMBAT_SKILLS;
-    public final ImmutableList<PrimarySkillType> GATHERING_SKILLS;
-    public final ImmutableList<PrimarySkillType> MISC_SKILLS;
+    public final @NotNull ImmutableList<String> LOCALIZED_SKILL_NAMES;
+    public final @NotNull ImmutableList<String> FORMATTED_SUBSKILL_NAMES;
+    public final @NotNull ImmutableSet<String> EXACT_SUBSKILL_NAMES;
+    public final @NotNull ImmutableList<PrimarySkillType> CHILD_SKILLS;
+    public final @NotNull ImmutableList<PrimarySkillType> NON_CHILD_SKILLS;
+    public final @NotNull ImmutableList<PrimarySkillType> COMBAT_SKILLS;
+    public final @NotNull ImmutableList<PrimarySkillType> GATHERING_SKILLS;
+    public final @NotNull ImmutableList<PrimarySkillType> MISC_SKILLS;
 
-    private ImmutableMap<SubSkillType, PrimarySkillType> subSkillParentRelationshipMap;
-    private ImmutableMap<SuperAbilityType, PrimarySkillType> superAbilityParentRelationshipMap; 
-    private ImmutableMap<PrimarySkillType, Set<SubSkillType>> primarySkillChildrenMap;
+    private final @NotNull ImmutableMap<SubSkillType, PrimarySkillType> subSkillParentRelationshipMap;
+    private final @NotNull ImmutableMap<SuperAbilityType, PrimarySkillType> superAbilityParentRelationshipMap;
+    private final @NotNull ImmutableMap<PrimarySkillType, Set<SubSkillType>> primarySkillChildrenMap;
 
     // The map below is for the super abilities which require readying a tool, its everything except blast mining
-    private ImmutableMap<PrimarySkillType, SuperAbilityType> mainActivatedAbilityChildMap; 
-    private ImmutableMap<PrimarySkillType, ToolType> primarySkillToolMap;
+    private final ImmutableMap<PrimarySkillType, SuperAbilityType> mainActivatedAbilityChildMap;
+    private final ImmutableMap<PrimarySkillType, ToolType> primarySkillToolMap;
 
     public SkillTools(@NotNull mcMMO pluginRef) {
         this.pluginRef = pluginRef;
 
-        initSubSkillRelationshipMap();
-        initPrimaryChildMap();
-        initPrimaryToolMap();
-        initSuperAbilityParentRelationships();
+        /*
+         * Setup subskill -> parent relationship map
+         */
+        EnumMap<SubSkillType, PrimarySkillType> tempSubParentMap = new EnumMap<SubSkillType, PrimarySkillType>(SubSkillType.class);
 
-        List<PrimarySkillType> childSkills = new ArrayList<>();
-        List<PrimarySkillType> nonChildSkills = new ArrayList<>();
+        //Super hacky and disgusting
+        for(PrimarySkillType primarySkillType1 : PrimarySkillType.values()) {
+            for(SubSkillType subSkillType : SubSkillType.values()) {
+                String[] splitSubSkillName = subSkillType.toString().split("_");
 
-        for (PrimarySkillType primarySkillType : PrimarySkillType.values()) {
-            if (isChildSkill(primarySkillType)) {
-                childSkills.add(primarySkillType);
-            } else {
-                nonChildSkills.add(primarySkillType);
+                if(primarySkillType1.toString().equalsIgnoreCase(splitSubSkillName[0])) {
+                    //Parent Skill Found
+                    tempSubParentMap.put(subSkillType, primarySkillType1);
+                }
             }
         }
 
-        COMBAT_SKILLS = ImmutableList.of(PrimarySkillType.ARCHERY, PrimarySkillType.AXES, PrimarySkillType.SWORDS, PrimarySkillType.TAMING, PrimarySkillType.UNARMED);
-        GATHERING_SKILLS = ImmutableList.of(PrimarySkillType.EXCAVATION, PrimarySkillType.FISHING, PrimarySkillType.HERBALISM, PrimarySkillType.MINING, PrimarySkillType.WOODCUTTING);
-        MISC_SKILLS = ImmutableList.of(PrimarySkillType.ACROBATICS, PrimarySkillType.ALCHEMY, PrimarySkillType.REPAIR, PrimarySkillType.SALVAGE, PrimarySkillType.SMELTING);
+        subSkillParentRelationshipMap = ImmutableMap.copyOf(tempSubParentMap);
 
-        LOCALIZED_SKILL_NAMES = ImmutableList.copyOf(buildLocalizedPrimarySkillNames());
-        FORMATTED_SUBSKILL_NAMES = ImmutableList.copyOf(buildFormattedSubSkillNameList());
-        EXACT_SUBSKILL_NAMES = ImmutableSet.copyOf(buildExactSubSkillNameList());
+        /*
+         * Setup primary -> (collection) subskill map
+         */
 
-        CHILD_SKILLS = ImmutableList.copyOf(childSkills);
-        NON_CHILD_SKILLS = ImmutableList.copyOf(nonChildSkills);
-    }
+        EnumMap<PrimarySkillType, Set<SubSkillType>> tempPrimaryChildMap = new EnumMap<PrimarySkillType, Set<SubSkillType>>(PrimarySkillType.class);
 
-    private void initPrimaryToolMap() {
+        //Init the empty Hash Sets
+        for(PrimarySkillType primarySkillType1 : PrimarySkillType.values()) {
+            tempPrimaryChildMap.put(primarySkillType1, new HashSet<>());
+        }
+
+        //Fill in the hash sets
+        for(SubSkillType subSkillType : SubSkillType.values()) {
+            PrimarySkillType parentSkill = subSkillParentRelationshipMap.get(subSkillType);
+
+            //Add this subskill as a child
+            tempPrimaryChildMap.get(parentSkill).add(subSkillType);
+        }
+
+        primarySkillChildrenMap = ImmutableMap.copyOf(tempPrimaryChildMap);
+
+        /*
+         * Setup primary -> tooltype map
+         */
         EnumMap<PrimarySkillType, ToolType> tempToolMap = new EnumMap<PrimarySkillType, ToolType>(PrimarySkillType.class);
 
         tempToolMap.put(PrimarySkillType.AXES, ToolType.AXE);
@@ -85,9 +99,12 @@ public class SkillTools {
         tempToolMap.put(PrimarySkillType.MINING, ToolType.PICKAXE);
 
         primarySkillToolMap = ImmutableMap.copyOf(tempToolMap);
-    }
 
-    private void initSuperAbilityParentRelationships() {
+        /*
+         * Setup ability -> primary map
+         * Setup primary -> ability map
+         */
+
         EnumMap<SuperAbilityType, PrimarySkillType> tempAbilityParentRelationshipMap = new EnumMap<SuperAbilityType, PrimarySkillType>(SuperAbilityType.class);
         EnumMap<PrimarySkillType, SuperAbilityType> tempMainActivatedAbilityChildMap = new EnumMap<PrimarySkillType, SuperAbilityType>(PrimarySkillType.class);
 
@@ -107,6 +124,40 @@ public class SkillTools {
 
         superAbilityParentRelationshipMap = ImmutableMap.copyOf(tempAbilityParentRelationshipMap);
         mainActivatedAbilityChildMap = ImmutableMap.copyOf(tempMainActivatedAbilityChildMap);
+
+        /*
+         * Build child skill and nonchild skill lists
+         */
+
+        List<PrimarySkillType> childSkills = new ArrayList<>();
+        List<PrimarySkillType> nonChildSkills = new ArrayList<>();
+
+        for (PrimarySkillType primarySkillType : PrimarySkillType.values()) {
+            if (isChildSkill(primarySkillType)) {
+                childSkills.add(primarySkillType);
+            } else {
+                nonChildSkills.add(primarySkillType);
+            }
+        }
+
+        CHILD_SKILLS = ImmutableList.copyOf(childSkills);
+        NON_CHILD_SKILLS = ImmutableList.copyOf(nonChildSkills);
+
+        /*
+         * Build categorized skill lists
+         */
+
+        COMBAT_SKILLS = ImmutableList.of(PrimarySkillType.ARCHERY, PrimarySkillType.AXES, PrimarySkillType.SWORDS, PrimarySkillType.TAMING, PrimarySkillType.UNARMED);
+        GATHERING_SKILLS = ImmutableList.of(PrimarySkillType.EXCAVATION, PrimarySkillType.FISHING, PrimarySkillType.HERBALISM, PrimarySkillType.MINING, PrimarySkillType.WOODCUTTING);
+        MISC_SKILLS = ImmutableList.of(PrimarySkillType.ACROBATICS, PrimarySkillType.ALCHEMY, PrimarySkillType.REPAIR, PrimarySkillType.SALVAGE, PrimarySkillType.SMELTING);
+
+        /*
+         * Build formatted/localized/etc string lists
+         */
+
+        LOCALIZED_SKILL_NAMES = ImmutableList.copyOf(buildLocalizedPrimarySkillNames());
+        FORMATTED_SUBSKILL_NAMES = ImmutableList.copyOf(buildFormattedSubSkillNameList());
+        EXACT_SUBSKILL_NAMES = ImmutableSet.copyOf(buildExactSubSkillNameList());
     }
 
     private @NotNull PrimarySkillType getSuperAbilityParent(SuperAbilityType superAbilityType) throws InvalidSkillException {
@@ -131,45 +182,6 @@ public class SkillTools {
         }
     }
 
-    /**
-     * Builds a list of localized {@link PrimarySkillType} names
-     * @return list of localized {@link PrimarySkillType} names
-     */
-    private @NotNull ArrayList<String> buildLocalizedPrimarySkillNames() {
-        ArrayList<String> localizedSkillNameList = new ArrayList<>();
-
-        for(PrimarySkillType primarySkillType : PrimarySkillType.values()) {
-            localizedSkillNameList.add(getLocalizedSkillName(primarySkillType));
-        }
-
-        Collections.sort(localizedSkillNameList);
-
-        return localizedSkillNameList;
-    }
-
-    /**
-     * Builds a map containing a HashSet of SubSkillTypes considered Children of PrimarySkillType
-     * Disgusting Hacky Fix until the new skill system is in place
-     */
-    private void initPrimaryChildMap() {
-        EnumMap<PrimarySkillType, Set<SubSkillType>> tempPrimaryChildMap = new EnumMap<PrimarySkillType, Set<SubSkillType>>(PrimarySkillType.class);
-
-        //Init the empty Hash Sets
-        for(PrimarySkillType primarySkillType : PrimarySkillType.values()) {
-            tempPrimaryChildMap.put(primarySkillType, new HashSet<>());
-        }
-
-        //Fill in the hash sets
-        for(SubSkillType subSkillType : SubSkillType.values()) {
-            PrimarySkillType parentSkill = subSkillParentRelationshipMap.get(subSkillType);
-
-            //Add this subskill as a child
-            tempPrimaryChildMap.get(parentSkill).add(subSkillType);
-        }
-
-        primarySkillChildrenMap = ImmutableMap.copyOf(tempPrimaryChildMap);
-    }
-
     /**
      * Makes a list of the "nice" version of sub skill names
      * Used in tab completion mostly
@@ -196,25 +208,20 @@ public class SkillTools {
     }
 
     /**
-     * Builds a map containing the relationships of SubSkillTypes to PrimarySkillTypes
-     * Disgusting Hacky Fix until the new skill system is in place
+     * Builds a list of localized {@link PrimarySkillType} names
+     * @return list of localized {@link PrimarySkillType} names
      */
-    private void initSubSkillRelationshipMap() {
-        EnumMap<SubSkillType, PrimarySkillType> tempSubParentMap = new EnumMap<SubSkillType, PrimarySkillType>(SubSkillType.class);
+    @VisibleForTesting
+    private @NotNull ArrayList<String> buildLocalizedPrimarySkillNames() {
+        ArrayList<String> localizedSkillNameList = new ArrayList<>();
 
-        //Super hacky and disgusting
         for(PrimarySkillType primarySkillType : PrimarySkillType.values()) {
-            for(SubSkillType subSkillType : SubSkillType.values()) {
-                String[] splitSubSkillName = subSkillType.toString().split("_");
-
-                if(primarySkillType.toString().equalsIgnoreCase(splitSubSkillName[0])) {
-                    //Parent Skill Found
-                    tempSubParentMap.put(subSkillType, primarySkillType);
-                }
-            }
+            localizedSkillNameList.add(getLocalizedSkillName(primarySkillType));
         }
 
-        subSkillParentRelationshipMap = ImmutableMap.copyOf(tempSubParentMap);
+        Collections.sort(localizedSkillNameList);
+
+        return localizedSkillNameList;
     }
 
     /**

src/test/java/com/gmail/nossr50/database/FlatFileDatabaseManagerTest.java

@@ -1,48 +1,62 @@
 package com.gmail.nossr50.database;
 
 import com.gmail.nossr50.TestUtil;
+import com.gmail.nossr50.datatypes.database.DatabaseType;
 import com.google.common.io.Files;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.junit.After;
-import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.powermock.modules.junit4.PowerMockRunner;
 
 import java.io.*;
+import java.util.List;
 import java.util.logging.Logger;
 
+import static org.junit.Assert.*;
+
 
 @RunWith(PowerMockRunner.class)
 public class FlatFileDatabaseManagerTest {
 
     public static final @NotNull String TEST_FILE_NAME = "test.mcmmo.users";
+    public static final int HEALTHY_RETURN_CODE = 0;
     private static File tempDir;
     private final static @NotNull Logger logger = Logger.getLogger(Logger.GLOBAL_LOGGER_NAME);
     private final long PURGE_TIME = 2630000000L;
-    private static @Nullable FlatFileDatabaseManager flatFileDatabaseManager;
+    private static @Nullable FlatFileDatabaseManager db;
 
     @Before
     public void init() {
+        assertNull(db);
         tempDir = Files.createTempDir();
-        flatFileDatabaseManager = new FlatFileDatabaseManager(tempDir.getPath() + File.separator + TEST_FILE_NAME, logger, PURGE_TIME, 0);
+        db = new FlatFileDatabaseManager(tempDir.getPath() + File.separator + TEST_FILE_NAME, logger, PURGE_TIME, 0);
     }
 
     @After
     public void tearDown() {
         TestUtil.recursiveDelete(tempDir);
-        flatFileDatabaseManager = null;
+        db = null;
     }
 
+    //Nothing wrong with this database
     private static String[] normalDatabaseData = {
             "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
             "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
             "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
     };
 
-    private static String[] splitDataBadDatabase = {
+    private static String[] corruptDatabaseData = {
+            "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
+            "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
+            "corruptdataboy:の:::ののの0:2452:0:1983:1937:1790:3042ののののの:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617のののののの583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:d20c6e8d-5615-4284-b8d1-e20b92011530:5:1600906906:",
+            "のjapaneseuserの:333:::0:2452:0:444:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:25870f0e-7558-4659-9f60-417e24cb3332:5:1600906906:",
+            "sameUUIDasjapaneseuser:333:::0:442:0:544:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:25870f0e-7558-4659-9f60-417e24cb3332:5:1600906906:",
+    };
+
+    private static String[] badDatabaseData = {
             //First entry here is missing some values
             "nossr50:1000:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
             //Second entry here has an integer value replaced by a string
@@ -51,12 +65,60 @@ public class FlatFileDatabaseManagerTest {
 
     @Test
     public void testPurgePowerlessUsers() {
-        Assert.assertNotNull(flatFileDatabaseManager);
-        addDataToFile(flatFileDatabaseManager, normalDatabaseData);
-        int purgeCount = flatFileDatabaseManager.purgePowerlessUsers();
-        Assert.assertEquals(purgeCount, 1); //1 User should have been purged
+        assertNotNull(db);
+        addDataToFile(db, normalDatabaseData);
+        int purgeCount = db.purgePowerlessUsers();
+        assertEquals(purgeCount, 1); //1 User should have been purged
     }
 
+    @Test
+    public void testCheckFileHealthAndStructure() {
+        assertNotNull(db);
+
+        addDataToFile(db, badDatabaseData);
+
+        List<FlatFileDataFlag> dataFlags = db.checkFileHealthAndStructure();
+        assertNotNull(dataFlags);
+        assertNotEquals(dataFlags.size(), 0);
+    }
+
+    @Test
+    public void testFindDuplicateNames() {
+
+    }
+
+    @Test
+    public void testFindDuplicateUUIDs() {
+
+    }
+
+    @Test
+    public void testFindCorruptData() {
+
+    }
+
+    @Test
+    public void testFindEmptyNames() {
+
+    }
+
+    @Test
+    public void testFindBadValues() {
+
+    }
+
+    @Test
+    public void testFindOutdatedData() {
+
+    }
+
+    @Test
+    public void testGetDatabaseType() {
+        assertNotNull(db);
+        assertEquals(db.getDatabaseType(), DatabaseType.FLATFILE);
+    }
+
+
     private void addDataToFile(@NotNull FlatFileDatabaseManager flatFileDatabaseManager, @NotNull String[] dataEntries) {
         String filePath = flatFileDatabaseManager.getUsersFile().getAbsolutePath();
         BufferedReader in = null;

src/test/java/com/gmail/nossr50/util/skills/SkillToolsTest.java

@@ -0,0 +1,16 @@
+//package com.gmail.nossr50.util.skills;
+//
+//import com.gmail.nossr50.datatypes.skills.PrimarySkillType;
+//import com.google.common.collect.ImmutableList;
+//import org.junit.Before;
+//import org.junit.Test;
+//import org.junit.runner.RunWith;
+//import org.powermock.core.classloader.annotations.PrepareForTest;
+//import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor;
+//import org.powermock.modules.junit4.PowerMockRunner;
+//
+//@RunWith(PowerMockRunner.class)
+//@PrepareForTest(SkillTools.class)
+//public class SkillToolsTest {
+//
+//}