EpicKnarvik97/mcMMO
EpicKnarvik97/mcMMO
1
0
Fork 0
mirror of https://github.com/mcMMO-Dev/mcMMO.git synced 2024-11-22 13:16:45 +01:00
Code Issues Projects Releases Wiki Activity

FlatFileDataProcessor will handle fixing and repairing the data

Browse Source
This commit is contained in:
nossr50 2021-04-12 12:55:31 -07:00
parent 60013c710b
commit 85f3221a60
11 changed files with 380 additions and 262 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main/java/com/gmail/nossr50/database/FlatFileDataContainer.java
View File

@ -1,5 +0,0 @@
package com.gmail.nossr50.database;
//Marker interface
public interface FlatFileDataContainer {
}

5
src/main/java/com/gmail/nossr50/database/FlatFileDataFlag.java
View File

@ -4,10 +4,9 @@ public enum FlatFileDataFlag {
INCOMPLETE, INCOMPLETE,
BAD_VALUES, BAD_VALUES,
MISSING_NAME, MISSING_NAME,
DUPLICATE_NAME_FIXABLE, DUPLICATE_NAME,
DUPLICATE_NAME_NOT_FIXABLE,
DUPLICATE_UUID, DUPLICATE_UUID,
MISSING_OR_NULL_UUID, BAD_UUID_DATA, //Can be because it is missing, null, or just not compatible data
TOO_INCOMPLETE, TOO_INCOMPLETE,
JUNK, JUNK,
EMPTY_LINE, EMPTY_LINE,

122
src/main/java/com/gmail/nossr50/database/FlatFileDataProcessor.java
View File

@ -1,30 +1,27 @@
package com.gmail.nossr50.database; package com.gmail.nossr50.database;
import com.gmail.nossr50.database.flatfile.CategorizedFlatFileData; import com.gmail.nossr50.database.flatfile.FlatFileDataBuilder;
import com.gmail.nossr50.database.flatfile.CategorizedFlatFileDataBuilder; import com.gmail.nossr50.database.flatfile.FlatFileDataContainer;
import com.gmail.nossr50.database.flatfile.FlatFileSaveDataProcessor;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import java.io.File;
import java.util.*; import java.util.*;
import java.util.logging.Logger; import java.util.logging.Logger;
import static com.gmail.nossr50.database.FlatFileDatabaseManager.*; import static com.gmail.nossr50.database.FlatFileDatabaseManager.*;
public class FlatFileDataProcessor { public class FlatFileDataProcessor {
public static final String INVALID_OLD_USERNAME = "_INVALID_OLD_USERNAME_"; private final @NotNull List<FlatFileDataContainer> flatFileDataContainers;
private @NotNull List<CategorizedFlatFileData> categorizedDataList; private final @NotNull List<FlatFileDataFlag> flatFileDataFlags;
private @NotNull List<FlatFileDataFlag> flatFileDataFlags;
private final @NotNull File userFile;
private final @NotNull Logger logger; private final @NotNull Logger logger;
private final HashSet<String> names; private final HashSet<String> names;
private final HashSet<UUID> uuids; private final HashSet<UUID> uuids;
private int uniqueProcessingID; private int uniqueProcessingID;
boolean corruptDataFound; boolean corruptDataFound;
public FlatFileDataProcessor(@NotNull File userFile, @NotNull Logger logger) { public FlatFileDataProcessor(@NotNull Logger logger) {
this.userFile = userFile;
this.logger = logger; this.logger = logger;
categorizedDataList = new ArrayList<>(); flatFileDataContainers = new ArrayList<>();
flatFileDataFlags = new ArrayList<>(); flatFileDataFlags = new ArrayList<>();
names = new HashSet<>(); names = new HashSet<>();
uuids = new HashSet<>(); uuids = new HashSet<>();
@ -32,16 +29,7 @@ public class FlatFileDataProcessor {
} }
public void processData(@NotNull String lineData) { public void processData(@NotNull String lineData) {
CategorizedFlatFileDataBuilder builder = new CategorizedFlatFileDataBuilder(lineData, uniqueProcessingID); assert !lineData.isEmpty();
uniqueProcessingID++;
/*
* Is the line empty?
*/
if (lineData.isEmpty()) {
registerData(builder.appendFlag(FlatFileDataFlag.EMPTY_LINE));
return;
}
//Make sure the data line is "correct" //Make sure the data line is "correct"
if(lineData.charAt(lineData.length() - 1) != ':') { if(lineData.charAt(lineData.length() - 1) != ':') {
@ -53,6 +41,11 @@ public class FlatFileDataProcessor {
//Split the data into an array //Split the data into an array
String[] splitDataLine = lineData.split(":"); String[] splitDataLine = lineData.split(":");
FlatFileDataBuilder builder = new FlatFileDataBuilder(splitDataLine, uniqueProcessingID);
uniqueProcessingID++;
boolean[] badDataValues = new boolean[DATA_ENTRY_COUNT];
boolean anyBadData = false;
//This is the minimum size of the split array needed to be considered proper data //This is the minimum size of the split array needed to be considered proper data
if(splitDataLine.length < getMinimumSplitDataLength()) { if(splitDataLine.length < getMinimumSplitDataLength()) {
//Data is considered junk //Data is considered junk
@ -82,7 +75,6 @@ public class FlatFileDataProcessor {
* Check for duplicate names * Check for duplicate names
*/ */
boolean nameIsDupe = false;
boolean invalidUUID = false; boolean invalidUUID = false;
String name = splitDataLine[USERNAME_INDEX]; String name = splitDataLine[USERNAME_INDEX];
@ -91,12 +83,17 @@ public class FlatFileDataProcessor {
if(name.isEmpty()) { if(name.isEmpty()) {
reportBadDataLine("No name found for data", "[MISSING NAME]", lineData); reportBadDataLine("No name found for data", "[MISSING NAME]", lineData);
builder.appendFlag(FlatFileDataFlag.MISSING_NAME); builder.appendFlag(FlatFileDataFlag.MISSING_NAME);
anyBadData = true;
badDataValues[USERNAME_INDEX] = true;
} }
if(strOfUUID.isEmpty() || strOfUUID.equalsIgnoreCase("NULL")) { if(strOfUUID.isEmpty() || strOfUUID.equalsIgnoreCase("NULL")) {
invalidUUID = true; invalidUUID = true;
badDataValues[UUID_INDEX] = true;
reportBadDataLine("Empty/null UUID for user", "Empty/null", lineData); reportBadDataLine("Empty/null UUID for user", "Empty/null", lineData);
builder.appendFlag(FlatFileDataFlag.MISSING_OR_NULL_UUID); builder.appendFlag(FlatFileDataFlag.BAD_UUID_DATA);
anyBadData = true;
} }
UUID uuid = null; UUID uuid = null;
@ -104,36 +101,23 @@ public class FlatFileDataProcessor {
try { try {
uuid = UUID.fromString(strOfUUID); uuid = UUID.fromString(strOfUUID);
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
invalidUUID = true;
//UUID does not conform //UUID does not conform
invalidUUID = true;
badDataValues[UUID_INDEX] = true;
reportBadDataLine("Invalid UUID data found for user", strOfUUID, lineData); reportBadDataLine("Invalid UUID data found for user", strOfUUID, lineData);
e.printStackTrace(); builder.appendFlag(FlatFileDataFlag.BAD_UUID_DATA);
} }
//Duplicate UUID is no good, reject them //Duplicate UUID is no good, reject them
if(uuid != null && uuids.contains(uuid)) { if(!invalidUUID && uuid != null && uuids.contains(uuid)) {
registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_UUID)); registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_UUID));
return; return;
} }
uuids.add(uuid); uuids.add(uuid);
if(names.contains(name)) { if(names.contains(name)) {
//Duplicate entry builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME);
nameIsDupe = true;
//We can accept them if they are a duped name if they have a unique UUID
if(invalidUUID) {
//Reject the data
reportBadDataLine("Duplicate user found and due to a missing UUID their data had to be discarded", name, lineData);
registerData(builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_NOT_FIXABLE));
return;
} else {
builder.appendFlag(FlatFileDataFlag.DUPLICATE_NAME_FIXABLE);
}
} }
if(!name.isEmpty()) if(!name.isEmpty())
@ -141,10 +125,17 @@ public class FlatFileDataProcessor {
//Make sure the data is up to date schema wise //Make sure the data is up to date schema wise
if(splitDataLine.length < DATA_ENTRY_COUNT) { if(splitDataLine.length < DATA_ENTRY_COUNT) {
splitDataLine = Arrays.copyOf(splitDataLine, DATA_ENTRY_COUNT+1); int oldLength = splitDataLine.length;
lineData = org.apache.commons.lang.StringUtils.join(splitDataLine, ":") + ":"; splitDataLine = Arrays.copyOf(splitDataLine, DATA_ENTRY_COUNT);
int newLength = splitDataLine.length;
//TODO: Test this
for(int i = oldLength; i < (newLength - 1); i++){
badDataValues[i] = true;
}
builder.appendFlag(FlatFileDataFlag.INCOMPLETE); builder.appendFlag(FlatFileDataFlag.INCOMPLETE);
builder.setStringDataRepresentation(lineData); builder.setSplitStringData(splitDataLine);
} }
/* /*
@ -153,9 +144,6 @@ public class FlatFileDataProcessor {
*/ */
//Check each data for bad values //Check each data for bad values
boolean[] badDataValues = new boolean[DATA_ENTRY_COUNT];
boolean anyBadData = false;
for(int i = 0; i < DATA_ENTRY_COUNT; i++) { for(int i = 0; i < DATA_ENTRY_COUNT; i++) {
if(shouldNotBeEmpty(splitDataLine[i], i)) { if(shouldNotBeEmpty(splitDataLine[i], i)) {
badDataValues[i] = true; badDataValues[i] = true;
@ -175,6 +163,7 @@ public class FlatFileDataProcessor {
if(anyBadData) { if(anyBadData) {
builder.appendFlag(FlatFileDataFlag.BAD_VALUES); builder.appendFlag(FlatFileDataFlag.BAD_VALUES);
builder.appendBadDataValues(badDataValues);
} }
registerData(builder); registerData(builder);
@ -240,13 +229,15 @@ public class FlatFileDataProcessor {
return UUID_INDEX + 1; return UUID_INDEX + 1;
} }
private void registerData(@NotNull CategorizedFlatFileDataBuilder builder) { private void registerData(@NotNull FlatFileDataBuilder builder) {
CategorizedFlatFileData categorizedFlatFileData = builder.build(); FlatFileDataContainer flatFileDataContainer = builder.build();
categorizedDataList.add(categorizedFlatFileData); flatFileDataContainers.add(flatFileDataContainer);
flatFileDataFlags.addAll(categorizedFlatFileData.getDataFlags());
if(flatFileDataContainer.getDataFlags() != null)
flatFileDataFlags.addAll(flatFileDataContainer.getDataFlags());
} }
public @NotNull ExpectedType getExpectedValueType(int dataIndex) { public @NotNull ExpectedType getExpectedValueType(int dataIndex) throws IndexOutOfBoundsException {
switch(dataIndex) { switch(dataIndex) {
case USERNAME_INDEX: case USERNAME_INDEX:
return ExpectedType.STRING; return ExpectedType.STRING;
@ -297,13 +288,13 @@ public class FlatFileDataProcessor {
return ExpectedType.FLOAT; return ExpectedType.FLOAT;
case UUID_INDEX: case UUID_INDEX:
return ExpectedType.UUID; return ExpectedType.UUID;
default:
return ExpectedType.OUT_OF_RANGE;
}
} }
public @NotNull List<CategorizedFlatFileData> getCategorizedDataList() { throw new IndexOutOfBoundsException();
return categorizedDataList; }
public @NotNull List<FlatFileDataContainer> getFlatFileDataContainers() {
return flatFileDataContainers;
} }
public @NotNull List<FlatFileDataFlag> getFlatFileDataFlags() { public @NotNull List<FlatFileDataFlag> getFlatFileDataFlags() {
@ -313,4 +304,23 @@ public class FlatFileDataProcessor {
public int getDataFlagCount() { public int getDataFlagCount() {
return flatFileDataFlags.size(); return flatFileDataFlags.size();
} }
public @NotNull StringBuilder processDataForSave() {
StringBuilder stringBuilder = new StringBuilder();
//Fix our data if needed and prepare it to be saved
for(FlatFileDataContainer dataContainer : flatFileDataContainers) {
String[] splitData = FlatFileSaveDataProcessor.getPreparedSaveDataLine(dataContainer);
if(splitData == null)
continue;
String fromSplit = org.apache.commons.lang.StringUtils.join(splitData, ":") + ":";
stringBuilder.append(fromSplit).append("\r\n");
}
return stringBuilder;
}
} }

152
src/main/java/com/gmail/nossr50/database/FlatFileDatabaseManager.java
View File

@ -20,6 +20,7 @@ import java.util.logging.Logger;
public final class FlatFileDatabaseManager implements DatabaseManager { public final class FlatFileDatabaseManager implements DatabaseManager {
public static final String IGNORED = "IGNORED"; public static final String IGNORED = "IGNORED";
public static final String LEGACY_INVALID_OLD_USERNAME = "_INVALID_OLD_USERNAME_'";
private final @NotNull EnumMap<PrimarySkillType, List<PlayerStat>> playerStatHash = new EnumMap<PrimarySkillType, List<PlayerStat>>(PrimarySkillType.class); private final @NotNull EnumMap<PrimarySkillType, List<PlayerStat>> playerStatHash = new EnumMap<PrimarySkillType, List<PlayerStat>>(PrimarySkillType.class);
private final @NotNull List<PlayerStat> powerLevels = new ArrayList<>(); private final @NotNull List<PlayerStat> powerLevels = new ArrayList<>();
private long lastUpdate = 0; private long lastUpdate = 0;
@ -965,24 +966,45 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
playerStatHash.put(PrimarySkillType.ALCHEMY, alchemy); playerStatHash.put(PrimarySkillType.ALCHEMY, alchemy);
} }
/**
* Makes sure that the users file has valid entries
* @return
*/
public @Nullable List<FlatFileDataFlag> checkFileHealthAndStructure() { public @Nullable List<FlatFileDataFlag> checkFileHealthAndStructure() {
FlatFileDataProcessor dataProcessor = null; FlatFileDataProcessor dataProcessor = null;
if (usersFile.exists()) { if (usersFile.exists()) {
BufferedReader bufferedReader = null; BufferedReader bufferedReader = null;
FileWriter fileWriter = null;
synchronized (fileWritingLock) { synchronized (fileWritingLock) {
dataProcessor = new FlatFileDataProcessor(usersFile, logger); dataProcessor = new FlatFileDataProcessor(logger);
try { try {
String currentLine; String currentLine;
bufferedReader = new BufferedReader(new FileReader(usersFilePath)); bufferedReader = new BufferedReader(new FileReader(usersFilePath));
//Analyze the data
while ((currentLine = bufferedReader.readLine()) != null) { while ((currentLine = bufferedReader.readLine()) != null) {
if(currentLine.isEmpty())
continue;
//TODO: We are never passing empty lines, should we remove the flag for them?
dataProcessor.processData(currentLine); dataProcessor.processData(currentLine);
} }
//Only update the file if needed
if(dataProcessor.getFlatFileDataFlags().size() > 0) {
logger.info("Saving the updated and or repaired FlatFile Database...");
fileWriter = new FileWriter(usersFilePath);
//Write data to file
fileWriter.write(dataProcessor.processDataForSave().toString());
}
} catch (IOException e) { } catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} finally {
closeResources(bufferedReader, fileWriter);
} }
} }
} }
@ -994,141 +1016,25 @@ public final class FlatFileDatabaseManager implements DatabaseManager {
} }
} }
private void closeResources(BufferedReader bufferedReader, FileWriter fileWriter) {
/** if(bufferedReader != null) {
* Checks that the file is present and valid
*/
public int checkFileHealthAndStructureOld() {
boolean corruptDataFound = false;
boolean oldDataFound = false;
if (usersFile.exists()) {
BufferedReader in = null;
FileWriter out = null;
synchronized (fileWritingLock) {
try { try {
bufferedReader.close();
in = new BufferedReader(new FileReader(usersFilePath));
StringBuilder writer = new StringBuilder();
String line;
HashSet<String> usernames = new HashSet<>();
HashSet<String> players = new HashSet<>();
while ((line = in.readLine()) != null) {
// Remove empty lines from the file
if (line.isEmpty()) {
continue;
}
// Length checks depend on last rawSplitData being ':'
if (line.charAt(line.length() - 1) != ':') {
line = line.concat(":");
}
String[] rawSplitData = line.split(":");
//Not enough data found to be considered a user reliably (NOTE: not foolproof)
if(rawSplitData.length < (UUID_INDEX + 1)) {
if(!corruptDataFound) {
logger.severe("Some corrupt data was found in mcmmo.users and has been repaired, it is possible that some player data has been lost in this process.");
corruptDataFound = true;
}
if(rawSplitData.length >= 10 //The value here is kind of arbitrary, it shouldn't be too low to avoid false positives, but also we aren't really going to correctly identify when player data has been corrupted or not with 100% accuracy ever
&& rawSplitData[0] != null && !rawSplitData[0].isEmpty()) {
if(rawSplitData[0].length() <= 16 && rawSplitData[0].length() >= 3) {
logger.severe("Not enough data found to recover corrupted player data for user: "+rawSplitData[0]);
}
}
//This user may have had a name so declare it
continue;
}
// Prevent the same username from being present multiple times
if (!usernames.add(rawSplitData[USERNAME_INDEX])) {
//TODO: Check if the commented out code was even necessary
rawSplitData[USERNAME_INDEX] = "_INVALID_OLD_USERNAME_'";
if (rawSplitData.length < UUID_INDEX + 1 || rawSplitData[UUID_INDEX].equals("NULL")) {
logger.severe("Fixing duplicate player names found in mcmmo.users");
continue;
}
}
// Prevent the same player from being present multiple times
if (rawSplitData.length >= (UUID_INDEX + 1) //TODO: Test this condition
&& (!rawSplitData[UUID_INDEX].isEmpty()
&& !rawSplitData[UUID_INDEX].equals("NULL") && !players.add(rawSplitData[UUID_INDEX]))) {
logger.severe("Removing duplicate player data from mcmmo.users");
logger.info("Duplicate Data: "+line);
continue;
}
//Correctly size the data (null entries for missing values)
if(line.length() < DATA_ENTRY_COUNT) { //TODO: Test this condition
oldDataFound = true;
String[] correctSizeSplitData = Arrays.copyOf(rawSplitData, DATA_ENTRY_COUNT);
line = org.apache.commons.lang.StringUtils.join(correctSizeSplitData, ":") + ":";
rawSplitData = line.split(":");
PlayerProfile temporaryProfile = loadFromLine(rawSplitData);
writeUserToLine(temporaryProfile, rawSplitData[USERNAME_INDEX], temporaryProfile.getUniqueId(), writer);
} else {
writer.append(line).append("\r\n");
}
}
// Write the new file
out = new FileWriter(usersFilePath);
out.write(writer.toString());
}
catch (IOException e) {
logger.severe("Exception while reading " + usersFilePath + " (Are you sure you formatted it correctly?)" + e);
}
finally {
if (in != null) {
try {
in.close();
} }
catch (IOException e) { catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
if (out != null) {
if (fileWriter != null) {
try { try {
out.close(); fileWriter.close();
} }
catch (IOException e) { catch (IOException e) {
e.printStackTrace(); e.printStackTrace();
} }
} }
} }
}
if(corruptDataFound)
logger.info("Corrupt data was found and removed, everything should be working fine. It is possible some player data was lost.");
}
usersFile.getParentFile().mkdir();
try {
logger.info("Creating mcmmo.users file...");
new File(usersFilePath).createNewFile();
}
catch (IOException e) {
e.printStackTrace();
}
if(corruptDataFound) {
return 1;
} else if(oldDataFound) {
return 2;
} else {
return 0;
}
}
private Integer getPlayerRank(String playerName, List<PlayerStat> statsList) { private Integer getPlayerRank(String playerName, List<PlayerStat> statsList) {
if (statsList == null) { if (statsList == null) {

42
src/main/java/com/gmail/nossr50/database/flatfile/BadCategorizedFlatFileData.java Normal file
View File

@ -0,0 +1,42 @@
package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataFlag;
import com.google.common.base.Objects;
import org.jetbrains.annotations.NotNull;
import java.util.Arrays;
import java.util.HashSet;
public class BadCategorizedFlatFileData extends CategorizedFlatFileData {
private final boolean[] badDataIndexes;
protected BadCategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String[] splitData, boolean[] badDataIndexes) {
super(uniqueProcessingId, dataFlags, splitData);
this.badDataIndexes = badDataIndexes;
}
public boolean[] getBadDataIndexes() {
return badDataIndexes;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
BadCategorizedFlatFileData that = (BadCategorizedFlatFileData) o;
return Objects.equal(badDataIndexes, that.badDataIndexes);
}
@Override
public int hashCode() {
return Objects.hashCode(super.hashCode(), badDataIndexes);
}
@Override
public String toString() {
return "BadCategorizedFlatFileData{" +
"badDataIndexes=" + Arrays.toString(badDataIndexes) +
'}';
}
}

44
src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileData.java
View File

@ -1,8 +1,7 @@
package com.gmail.nossr50.database.flatfile; package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataContainer;
import com.gmail.nossr50.database.FlatFileDataFlag; import com.gmail.nossr50.database.FlatFileDataFlag;
import com.gmail.nossr50.database.FlatFileDatabaseManager; import com.google.common.base.Objects;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import java.util.HashSet; import java.util.HashSet;
@ -10,30 +9,21 @@ import java.util.Set;
public class CategorizedFlatFileData implements FlatFileDataContainer { public class CategorizedFlatFileData implements FlatFileDataContainer {
private final @NotNull Set<FlatFileDataFlag> dataFlags; private final @NotNull Set<FlatFileDataFlag> dataFlags;
private final @NotNull String stringDataRepresentation; private final @NotNull String[] splitData;
private final int uniqueProcessingId; private final int uniqueProcessingId;
private final boolean[] badDataIndexes;
protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation) { protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String[] splitData) {
this.uniqueProcessingId = uniqueProcessingId; this.uniqueProcessingId = uniqueProcessingId;
this.dataFlags = dataFlags; this.dataFlags = dataFlags;
this.stringDataRepresentation = stringDataRepresentation; this.splitData = splitData;
badDataIndexes = new boolean[FlatFileDatabaseManager.DATA_ENTRY_COUNT];
}
protected CategorizedFlatFileData(int uniqueProcessingId, @NotNull HashSet<FlatFileDataFlag> dataFlags, @NotNull String stringDataRepresentation, boolean[] badDataIndexes) {
this.uniqueProcessingId = uniqueProcessingId;
this.dataFlags = dataFlags;
this.stringDataRepresentation = stringDataRepresentation;
this.badDataIndexes = badDataIndexes;
} }
public @NotNull Set<FlatFileDataFlag> getDataFlags() { public @NotNull Set<FlatFileDataFlag> getDataFlags() {
return dataFlags; return dataFlags;
} }
public @NotNull String getStringDataRepresentation() { public @NotNull String[] getSplitData() {
return stringDataRepresentation; return splitData;
} }
public int getUniqueProcessingId() { public int getUniqueProcessingId() {
@ -44,7 +34,25 @@ public class CategorizedFlatFileData implements FlatFileDataContainer {
return dataFlags.size() == 0; return dataFlags.size() == 0;
} }
public boolean[] getBadDataIndexes() { @Override
return badDataIndexes; public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
CategorizedFlatFileData that = (CategorizedFlatFileData) o;
return uniqueProcessingId == that.uniqueProcessingId && Objects.equal(dataFlags, that.dataFlags) && Objects.equal(splitData, that.splitData);
}
@Override
public int hashCode() {
return Objects.hashCode(dataFlags, splitData, uniqueProcessingId);
}
@Override
public String toString() {
return "CategorizedFlatFileData{" +
"dataFlags=" + dataFlags +
", stringDataRepresentation='" + splitData + '\'' +
", uniqueProcessingId=" + uniqueProcessingId +
'}';
} }
} }

32
src/main/java/com/gmail/nossr50/database/flatfile/CategorizedFlatFileDataBuilder.java
View File

@ -1,32 +0,0 @@
package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataFlag;
import org.jetbrains.annotations.NotNull;
import java.util.HashSet;
public class CategorizedFlatFileDataBuilder {
private final @NotNull HashSet<FlatFileDataFlag> dataFlags;
private @NotNull String stringDataRepresentation;
private final int uniqueProcessingId;
public CategorizedFlatFileDataBuilder(@NotNull String stringDataRepresentation, int uniqueProcessingId) {
this.uniqueProcessingId = uniqueProcessingId;
this.stringDataRepresentation = stringDataRepresentation;
dataFlags = new HashSet<>();
}
public CategorizedFlatFileDataBuilder appendFlag(@NotNull FlatFileDataFlag dataFlag) {
dataFlags.add(dataFlag);
return this;
}
public CategorizedFlatFileData build() {
return new CategorizedFlatFileData(uniqueProcessingId, dataFlags, stringDataRepresentation);
}
public CategorizedFlatFileDataBuilder setStringDataRepresentation(@NotNull String stringDataRepresentation) {
this.stringDataRepresentation = stringDataRepresentation;
return this;
}
}

42
src/main/java/com/gmail/nossr50/database/flatfile/FlatFileDataBuilder.java Normal file
View File

@ -0,0 +1,42 @@
package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataFlag;
import org.jetbrains.annotations.NotNull;
import java.util.HashSet;
public class FlatFileDataBuilder {
private final @NotNull HashSet<FlatFileDataFlag> dataFlags;
private @NotNull String[] splitStringData;
private final int uniqueProcessingId;
private boolean[] badDataValues;
public FlatFileDataBuilder(@NotNull String[] splitStringData, int uniqueProcessingId) {
this.uniqueProcessingId = uniqueProcessingId;
this.splitStringData = splitStringData;
dataFlags = new HashSet<>();
}
public @NotNull FlatFileDataBuilder appendFlag(@NotNull FlatFileDataFlag dataFlag) {
dataFlags.add(dataFlag);
return this;
}
public @NotNull FlatFileDataBuilder appendBadDataValues(boolean[] badDataValues) {
this.badDataValues = badDataValues;
return this;
}
public @NotNull FlatFileDataContainer build() {
if(dataFlags.contains(FlatFileDataFlag.BAD_VALUES)) {
return new BadCategorizedFlatFileData(uniqueProcessingId, dataFlags, splitStringData, badDataValues);
}
return new CategorizedFlatFileData(uniqueProcessingId, dataFlags, splitStringData);
}
public @NotNull FlatFileDataBuilder setSplitStringData(@NotNull String[] splitStringData) {
this.splitStringData = splitStringData;
return this;
}
}

21
src/main/java/com/gmail/nossr50/database/flatfile/FlatFileDataContainer.java Normal file
View File

@ -0,0 +1,21 @@
package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataFlag;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.Set;
public interface FlatFileDataContainer {
default @Nullable Set<FlatFileDataFlag> getDataFlags() {
return null;
}
@NotNull String[] getSplitData();
int getUniqueProcessingId();
default boolean isHealthyData() {
return getDataFlags() == null || getDataFlags().size() == 0;
}
}

117
src/main/java/com/gmail/nossr50/database/flatfile/FlatFileSaveDataProcessor.java Normal file
View File

@ -0,0 +1,117 @@
package com.gmail.nossr50.database.flatfile;
import com.gmail.nossr50.database.FlatFileDataFlag;
import com.gmail.nossr50.database.FlatFileDatabaseManager;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import static com.gmail.nossr50.database.FlatFileDatabaseManager.*;
import static com.gmail.nossr50.database.FlatFileDatabaseManager.UUID_INDEX;
public class FlatFileSaveDataProcessor {
public static @Nullable String[] getPreparedSaveDataLine(@NotNull FlatFileDataContainer dataContainer) {
if(dataContainer.getDataFlags() == null) {
return dataContainer.getSplitData();
}
//Data of this type is not salvageable
//TODO: Test that we ignore the things we are supposed to ignore
//TODO: Should we even keep track of the bad data or just not even build data containers for it? Making containers for it is only really useful for debugging.. well I suppose operations are typically async so it shouldn't matter
if(dataContainer.getDataFlags().contains(FlatFileDataFlag.JUNK)
|| dataContainer.getDataFlags().contains(FlatFileDataFlag.DUPLICATE_UUID) //For now we will not try to fix any issues with UUIDs
|| dataContainer.getDataFlags().contains(FlatFileDataFlag.BAD_UUID_DATA) //For now we will not try to fix any issues with UUIDs
|| dataContainer.getDataFlags().contains(FlatFileDataFlag.TOO_INCOMPLETE)
|| dataContainer.getDataFlags().contains(FlatFileDataFlag.EMPTY_LINE)) {
return null;
}
String[] splitData;
/*
* First fix the bad data values if they exist
*/
if(dataContainer instanceof BadCategorizedFlatFileData) {
BadCategorizedFlatFileData badData = (BadCategorizedFlatFileData) dataContainer;
splitData = repairBadData(dataContainer.getSplitData(), badData.getBadDataIndexes());
} else {
splitData = dataContainer.getSplitData();
}
//Make sure we have as many values as we are supposed to
assert splitData.length == FlatFileDatabaseManager.DATA_ENTRY_COUNT;
return splitData;
}
public static @NotNull String[] repairBadData(@NotNull String[] splitData, boolean[] badDataValues) {
for(int i = 0; i < FlatFileDatabaseManager.DATA_ENTRY_COUNT; i++) {
if(badDataValues[i]) {
//This data value was marked as bad so we zero initialize it
splitData[i] = getZeroInitialisedData(i, 0);
}
}
return splitData;
}
/**
* @param index "zero" Initialization will depend on what the index is for
* @return the "zero" initialized data corresponding to the index
*/
public static @NotNull String getZeroInitialisedData(int index, int startingLevel) throws IndexOutOfBoundsException {
switch(index) {
case USERNAME_INDEX:
return LEGACY_INVALID_OLD_USERNAME; //We'll keep using this value for legacy compatibility reasons (not sure if needed but don't care)
case 2: //Assumption: Used to be for something, no longer used
case 3: //Assumption: Used to be for something, no longer used
case 23: //Assumption: Used to be used for something, no longer used
case 33: //Assumption: Used to be used for something, no longer used
case HEALTHBAR:
return "IGNORED";
case SKILLS_MINING:
case SKILLS_REPAIR:
case SKILLS_UNARMED:
case SKILLS_HERBALISM:
case SKILLS_EXCAVATION:
case SKILLS_ARCHERY:
case SKILLS_SWORDS:
case SKILLS_AXES:
case SKILLS_WOODCUTTING:
case SKILLS_ACROBATICS:
case SKILLS_TAMING:
case SKILLS_FISHING:
case SKILLS_ALCHEMY:
return String.valueOf(startingLevel);
case LAST_LOGIN:
return String.valueOf(System.currentTimeMillis() / 1000); //This is just to shorten the value
case COOLDOWN_BERSERK:
case COOLDOWN_GIGA_DRILL_BREAKER:
case COOLDOWN_TREE_FELLER:
case COOLDOWN_GREEN_TERRA:
case COOLDOWN_SERRATED_STRIKES:
case COOLDOWN_SKULL_SPLITTER:
case COOLDOWN_SUPER_BREAKER:
case COOLDOWN_BLAST_MINING:
case SCOREBOARD_TIPS:
case COOLDOWN_CHIMAERA_WING:
case EXP_MINING:
case EXP_WOODCUTTING:
case EXP_REPAIR:
case EXP_UNARMED:
case EXP_HERBALISM:
case EXP_EXCAVATION:
case EXP_ARCHERY:
case EXP_SWORDS:
case EXP_AXES:
case EXP_ACROBATICS:
case EXP_TAMING:
case EXP_FISHING:
case EXP_ALCHEMY:
return "0";
case UUID_INDEX:
throw new IndexOutOfBoundsException(); //TODO: Add UUID recovery? Might not even be worth it.
}
throw new IndexOutOfBoundsException();
}
}

46
src/test/java/com/gmail/nossr50/database/FlatFileDatabaseManagerTest.java
View File

@ -48,28 +48,34 @@ public class FlatFileDatabaseManagerTest {
"powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:" "powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
}; };
private static final String[] badUUIDDatabaseData = {
"nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
"z750:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:3:5:1600906906:", //This one has an incorrect UUID representation
"powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
};
private static final String[] outdatedDatabaseData = { private static final String[] outdatedDatabaseData = {
"nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:", "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
"mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:", "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
"powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:" //This user is missing data added after UUID index "electronicboy:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:" //This user is missing data added after UUID index
}; };
private static final String[] emptyLineDatabaseData = { private static final String[] emptyLineDatabaseData = {
"nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:", "nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
"mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:", "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
"powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:", "kashike:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:",
"" //EMPTY LINE "" //EMPTY LINE
}; };
private static final String[] emptyNameDatabaseData = { private static final String[] emptyNameDatabaseData = {
":1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:", ":1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
"mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:", "mrfloris:2420:::0:2452:0:1983:1937:1790:3042:1138:3102:2408:3411:0:0:0:0:0:0:0:0::642:0:1617583171:0:1617165043:0:1617583004:1617563189:1616785408::2184:0:0:1617852413:HEARTS:415:0:631e3896-da2a-4077-974b-d047859d76bc:5:1600906906:",
"powerless:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:" "aikar:0:::0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0:0::0:0:0:0:0:0:0:0:0::0:0:0:0:HEARTS:0:0:e0d07db8-f7e8-43c7-9ded-864dfc6f3b7c:5:1600906906:"
}; };
private static final String[] duplicateNameDatabaseData = { private static final String[] duplicateNameDatabaseData = {
"nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:", "mochi:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:588fe472-1c82-4c4e-9aa1-7eefccb277e3:0:0:",
"nossr50:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:631e3896-da2a-4077-974b-d047859d76bc:0:0:", "mochi:1000:::0:1000:640:1000:1000:1000:1000:1000:1000:1000:1000:16:0:500:0:0:0:0:0::1000:0:0:0:1593543012:0:0:0:0::1000:0:0:1593806053:HEARTS:1000:0:631e3896-da2a-4077-974b-d047859d76bc:0:0:",
}; };
private static final String[] duplicateUUIDDatabaseData = { private static final String[] duplicateUUIDDatabaseData = {
@ -109,38 +115,43 @@ public class FlatFileDatabaseManagerTest {
} }
@Test @Test
public void testFindDuplicateNames() { public void testFindFixableDuplicateNames() {
addDataAndCheckForFlag(db, duplicateNameDatabaseData, FlatFileDataFlag.DUPLICATE_NAME_FIXABLE); overwriteDataAndCheckForFlag(db, duplicateNameDatabaseData, FlatFileDataFlag.DUPLICATE_NAME);
} }
@Test @Test
public void testFindDuplicateUUIDs() { public void testFindDuplicateUUIDs() {
addDataAndCheckForFlag(db, duplicateUUIDDatabaseData, FlatFileDataFlag.DUPLICATE_UUID); overwriteDataAndCheckForFlag(db, duplicateUUIDDatabaseData, FlatFileDataFlag.DUPLICATE_UUID);
}
@Test()
public void findBadUUIDData() {
overwriteDataAndCheckForFlag(db, badUUIDDatabaseData, FlatFileDataFlag.BAD_UUID_DATA);
} }
@Test @Test
public void testFindCorruptData() { public void testFindCorruptData() {
addDataAndCheckForFlag(db, corruptDatabaseData, FlatFileDataFlag.JUNK); overwriteDataAndCheckForFlag(db, corruptDatabaseData, FlatFileDataFlag.JUNK);
} }
@Test @Test
public void testFindEmptyNames() { public void testFindEmptyNames() {
addDataAndCheckForFlag(db, emptyNameDatabaseData, FlatFileDataFlag.MISSING_NAME); overwriteDataAndCheckForFlag(db, emptyNameDatabaseData, FlatFileDataFlag.MISSING_NAME);
} }
@Test // @Test
public void testFindEmptyLine() { // public void testFindEmptyLine() {
addDataAndCheckForFlag(db, emptyLineDatabaseData, FlatFileDataFlag.EMPTY_LINE); // overwriteDataAndCheckForFlag(db, emptyLineDatabaseData, FlatFileDataFlag.EMPTY_LINE);
} // }
@Test @Test
public void testFindBadValues() { public void testFindBadValues() {
addDataAndCheckForFlag(db, badDatabaseData, FlatFileDataFlag.BAD_VALUES); overwriteDataAndCheckForFlag(db, badDatabaseData, FlatFileDataFlag.BAD_VALUES);
} }
@Test @Test
public void testFindOutdatedData() { public void testFindOutdatedData() {
addDataAndCheckForFlag(db, outdatedDatabaseData, FlatFileDataFlag.INCOMPLETE); overwriteDataAndCheckForFlag(db, outdatedDatabaseData, FlatFileDataFlag.INCOMPLETE);
} }
@Test @Test
@ -149,7 +160,6 @@ public class FlatFileDatabaseManagerTest {
assertEquals(db.getDatabaseType(), DatabaseType.FLATFILE); assertEquals(db.getDatabaseType(), DatabaseType.FLATFILE);
} }
private void replaceDataInFile(@NotNull FlatFileDatabaseManager flatFileDatabaseManager, @NotNull String[] dataEntries) { private void replaceDataInFile(@NotNull FlatFileDatabaseManager flatFileDatabaseManager, @NotNull String[] dataEntries) {
String filePath = flatFileDatabaseManager.getUsersFile().getAbsolutePath(); String filePath = flatFileDatabaseManager.getUsersFile().getAbsolutePath();
BufferedReader in = null; BufferedReader in = null;
@ -203,7 +213,7 @@ public class FlatFileDatabaseManagerTest {
} }
private void addDataAndCheckForFlag(@NotNull FlatFileDatabaseManager targetDatabase, @NotNull String[] data, @NotNull FlatFileDataFlag flag) { private void overwriteDataAndCheckForFlag(@NotNull FlatFileDatabaseManager targetDatabase, @NotNull String[] data, @NotNull FlatFileDataFlag flag) {
replaceDataInFile(targetDatabase, data); replaceDataInFile(targetDatabase, data);
List<FlatFileDataFlag> dataFlags = targetDatabase.checkFileHealthAndStructure(); List<FlatFileDataFlag> dataFlags = targetDatabase.checkFileHealthAndStructure();