Fixed several SQL bugs + SQL db is now fully UTF-8 compatible

Changelog.txt

@@ -1,5 +1,7 @@
 Version 2.1.180
     mcMMO now uses UTF-8 compliant encoding for SQL databases (utf8mb4)
+    Fixed a bug where mcMMO could in some circumstances fail to update SQL schema and mark it as successful
+    Renamed updates.yml to updates_overhaul.yml to avoid some potential issues when upgrading from classic
 
 Version 2.1.179
     Fixed a bug for FlatFile databases where some players with changed nicknames would have their levels not loaded upon login (possibly wiping their data)

src/main/java/com/gmail/nossr50/database/SQLDatabaseManager.java

@@ -26,6 +26,9 @@ import java.util.concurrent.locks.ReentrantLock;
 
 public final class SQLDatabaseManager implements DatabaseManager {
     private static final String ALL_QUERY_VERSION = "total";
+    public static final String MOBHEALTHBAR_VARCHAR = "VARCHAR(50)";
+    public static final String UUID_VARCHAR = "VARCHAR(36)";
+    public static final String USER_VARCHAR = "VARCHAR(40)";
     private final String tablePrefix = Config.getInstance().getMySQLTablePrefix();
 
     private final Map<UUID, Integer> cachedUserIDs = new HashMap<>();
@@ -38,7 +41,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
 
     private final ReentrantLock massUpdateLock = new ReentrantLock();
 
-    private final String ENCODING = "utf8mb4"; //This is compliant with UTF-8 while "utf8" is not, confusing but this is how it is.
+    private final String CHARSET_SQL = "utf8mb4"; //This is compliant with UTF-8 while "utf8" is not, confusing but this is how it is.
 
     protected SQLDatabaseManager() {
         String connectionString = "jdbc:mysql://" + Config.getInstance().getMySQLServerName()
@@ -816,7 +819,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
                     + "`lastlogin` int(32) unsigned NOT NULL,"
                     + "PRIMARY KEY (`id`),"
                     + "INDEX(`user`(20) ASC),"
-                    + "UNIQUE KEY `uuid` (`uuid`)) DEFAULT CHARSET=" + ENCODING + " AUTO_INCREMENT=1;");
+                    + "UNIQUE KEY `uuid` (`uuid`)) DEFAULT CHARSET=" + CHARSET_SQL + " AUTO_INCREMENT=1;");
                 tryClose(createStatement);
             }
             tryClose(resultSet);
@@ -830,7 +833,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
                         + "`mobhealthbar` varchar(50) NOT NULL DEFAULT '" + Config.getInstance().getMobHealthbarDefault() + "',"
                         + "`scoreboardtips` int(10) NOT NULL DEFAULT '0',"
                         + "PRIMARY KEY (`user_id`)) "
-                        + "DEFAULT CHARSET=" + ENCODING + ";");
+                        + "DEFAULT CHARSET=" + CHARSET_SQL + ";");
                 tryClose(createStatement);
             }
             tryClose(resultSet);
@@ -855,7 +858,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
                         + "`blast_mining` int(32) unsigned NOT NULL DEFAULT '0',"
                         + "`chimaera_wing` int(32) unsigned NOT NULL DEFAULT '0',"
                         + "PRIMARY KEY (`user_id`)) "
-                        + "DEFAULT CHARSET=" + ENCODING + ";");
+                        + "DEFAULT CHARSET=" + CHARSET_SQL + ";");
                 tryClose(createStatement);
             }
             tryClose(resultSet);
@@ -883,7 +886,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
                         + "`alchemy` int(10) unsigned NOT NULL DEFAULT "+startingLevel+","
                         + "`total` int(10) unsigned NOT NULL DEFAULT "+totalLevel+","
                         + "PRIMARY KEY (`user_id`)) "
-                        + "DEFAULT CHARSET=" + ENCODING + ";");
+                        + "DEFAULT CHARSET=" + CHARSET_SQL + ";");
                 tryClose(createStatement);
             }
             tryClose(resultSet);
@@ -908,7 +911,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
                         + "`fishing` int(10) unsigned NOT NULL DEFAULT '0',"
                         + "`alchemy` int(10) unsigned NOT NULL DEFAULT '0',"
                         + "PRIMARY KEY (`user_id`)) "
-                        + "DEFAULT CHARSET=" + ENCODING + ";");
+                        + "DEFAULT CHARSET=" + CHARSET_SQL + ";");
                 tryClose(createStatement);
             }
             tryClose(resultSet);
@@ -1031,12 +1034,14 @@ public final class SQLDatabaseManager implements DatabaseManager {
                     checkUpgradeAddUniqueChimaeraWing(statement);
                     break;
 
+                case SQL_CHARSET_UTF8MB4:
+                    updateCharacterSet(statement);
+                    break;
+
                 default:
                     break;
 
             }
-
-            mcMMO.getUpgradeManager().setUpgradeCompleted(upgrade);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1193,6 +1198,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
             statement.execute("ALTER TABLE `" + tablePrefix + "users` " 
                     + "DROP INDEX `user`,"
                     + "ADD INDEX `user` (`user`(20) ASC)");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_NAME_UNIQUENESS);
         } catch (SQLException ex) {
             ex.printStackTrace();
         } finally {
@@ -1203,6 +1209,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddAlchemy(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `alchemy` FROM `" + tablePrefix + "skills` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_ALCHEMY);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Alchemy...");
@@ -1214,6 +1221,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddBlastMiningCooldown(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `blast_mining` FROM `" + tablePrefix + "cooldowns` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_BLAST_MINING_COOLDOWN);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Blast Mining...");
@@ -1224,6 +1232,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddUniqueChimaeraWing(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `chimaera_wing` FROM `" + tablePrefix + "cooldowns` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_UNIQUE_PLAYER_DATA);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Chimaera Wing...");
@@ -1234,6 +1243,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddFishing(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `fishing` FROM `" + tablePrefix + "skills` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_FISHING);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for Fishing...");
@@ -1245,6 +1255,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddMobHealthbars(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `mobhealthbar` FROM `" + tablePrefix + "huds` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_MOB_HEALTHBARS);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for mob healthbars...");
@@ -1255,6 +1266,7 @@ public final class SQLDatabaseManager implements DatabaseManager {
     private void checkUpgradeAddScoreboardTips(final Statement statement) throws SQLException {
         try {
             statement.executeQuery("SELECT `scoreboardtips` FROM `" + tablePrefix + "huds` LIMIT 1");
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SCOREBOARD_TIPS);
         }
         catch (SQLException ex) {
             mcMMO.p.getLogger().info("Updating mcMMO MySQL tables for scoreboard tips...");
@@ -1283,6 +1295,8 @@ public final class SQLDatabaseManager implements DatabaseManager {
                     }
                 }
             }
+
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SQL_INDEXES);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1313,6 +1327,8 @@ public final class SQLDatabaseManager implements DatabaseManager {
                 statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` ADD `uuid` varchar(36) NULL DEFAULT NULL");
                 statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` ADD UNIQUE INDEX `uuid` (`uuid`) USING BTREE");
             }
+
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_UUIDS);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1379,6 +1395,8 @@ public final class SQLDatabaseManager implements DatabaseManager {
                 mcMMO.p.getLogger().info("Removing party name from users table...");
                 statement.executeUpdate("ALTER TABLE `" + tablePrefix + "users` DROP COLUMN `party`");
             }
+
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_SQL_PARTY_NAMES);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1414,6 +1432,8 @@ public final class SQLDatabaseManager implements DatabaseManager {
                 statement.executeUpdate("ALTER TABLE `" + tablePrefix + "skills` ADD INDEX `idx_total` (`total`) USING BTREE");
                 connection.commit();
             }
+
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.ADD_SKILL_TOTAL);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1445,6 +1465,8 @@ public final class SQLDatabaseManager implements DatabaseManager {
                 mcMMO.p.getLogger().info("Removing Spout HUD type from huds table...");
                 statement.executeUpdate("ALTER TABLE `" + tablePrefix + "huds` DROP COLUMN `hudtype`");
             }
+
+            mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.DROP_SPOUT);
         }
         catch (SQLException ex) {
             printErrors(ex);
@@ -1557,4 +1579,67 @@ public final class SQLDatabaseManager implements DatabaseManager {
             tryClose(connection);
         }
     }
+
+    private void updateCharacterSet(@NotNull Statement statement) {
+        //TODO: Could check the tables for being latin1 before executing queries but it seems moot because it is likely the same computational effort
+        /*
+            The following columns were set to use latin1 historically (now utf8mb4)
+            column user in <tablePrefix>users
+            column uuid in <tablePrefix>users
+
+            column mobhealthbar in <tablePrefix>huds
+         */
+
+        //Alter users table
+        mcMMO.p.getLogger().info("SQL Converting tables from latin1 to utf8mb4");
+
+        //Update "user" column
+        try {
+        mcMMO.p.getLogger().info("Updating user column to new encoding");
+        statement.executeUpdate(getUpdateUserInUsersTableSQLQuery());
+
+        //Update "uuid" column
+        mcMMO.p.getLogger().info("Updating user column to new encoding");
+        statement.executeUpdate(getUpdateUUIDInUsersTableSQLQuery());
+
+        //Update "mobhealthbar" column
+        mcMMO.p.getLogger().info("Updating mobhealthbar column to new encoding");
+        statement.executeUpdate(getUpdateMobHealthBarInHudsTableSQLQuery());
+
+        mcMMO.getUpgradeManager().setUpgradeCompleted(UpgradeType.SQL_CHARSET_UTF8MB4);
+
+        } catch (SQLException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @NotNull
+    private String getUpdateUserInUsersTableSQLQuery() {
+        return "ALTER TABLE\n" +
+                "    " + tablePrefix + "users\n" +
+                "    CHANGE user user\n" +
+                "    " + USER_VARCHAR + "\n" +
+                "    CHARACTER SET utf8mb4\n" +
+                "    COLLATE utf8mb4_unicode_ci;";
+    }
+
+    @NotNull
+    private String getUpdateUUIDInUsersTableSQLQuery() {
+        return "ALTER TABLE\n" +
+                "    " + tablePrefix + "users\n" +
+                "    CHANGE uuid uuid\n" +
+                "    " + UUID_VARCHAR + "\n" +
+                "    CHARACTER SET utf8mb4\n" +
+                "    COLLATE utf8mb4_unicode_ci;";
+    }
+
+    @NotNull
+    private String getUpdateMobHealthBarInHudsTableSQLQuery() {
+        return "ALTER TABLE\n" +
+                "    " + tablePrefix + "huds\n" +
+                "    CHANGE mobhealthbar mobhealthbar\n" +
+                "    " + MOBHEALTHBAR_VARCHAR + "\n" +
+                "    CHARACTER SET utf8mb4\n" +
+                "    COLLATE utf8mb4_unicode_ci;";
+    }
 }

src/main/java/com/gmail/nossr50/datatypes/database/UpgradeType.java

@@ -16,5 +16,6 @@ public enum UpgradeType {
     ADD_UNIQUE_PLAYER_DATA,
     FIX_SPELLING_NETHERITE_SALVAGE,
     FIX_SPELLING_NETHERITE_REPAIR,
-    FIX_NETHERITE_SALVAGE_QUANTITIES
+    FIX_NETHERITE_SALVAGE_QUANTITIES,
+    SQL_CHARSET_UTF8MB4
 }

src/main/java/com/gmail/nossr50/util/upgrade/UpgradeManager.java

@@ -11,7 +11,7 @@ public class UpgradeManager extends ConfigLoader {
     private final Set<UpgradeType> setNeededUpgrades;
 
     public UpgradeManager() {
-        super("upgrades.yml");
+        super("upgrades_overhaul.yml"); //overhaul is added so we don't have any issues with classic
 
         setNeededUpgrades = EnumSet.allOf(UpgradeType.class);
 

src/main/resources/upgrades_overhaul.yml

@@ -11,3 +11,4 @@ Upgrades_Finished:
     FIX_SPELLING_NETHERITE_REPAIR: false
     FIX_NETHERITE_SALVAGE_QUANTITIES: false
     ADD_UUIDS: false
+    SQL_CHARSET_UTF8MB4: false