diff --git a/src/main/java/net/knarcraft/minstrel/command/AddSongCommand.java b/src/main/java/net/knarcraft/minstrel/command/AddSongCommand.java
index c26019f..693f00e 100644
--- a/src/main/java/net/knarcraft/minstrel/command/AddSongCommand.java
+++ b/src/main/java/net/knarcraft/minstrel/command/AddSongCommand.java
@@ -65,6 +65,7 @@ public class AddSongCommand implements CommandExecutor {
         Song song = new Song(category, songIdString, duration);
         playlist.addSong(song);
 
+        //If this is the first song in the playlist, start playing
         if (playlist.getSongs().size() == 1) {
             playlist.play(minstrelTrait);
         }
diff --git a/src/main/java/net/knarcraft/minstrel/command/RemoveSongCommand.java b/src/main/java/net/knarcraft/minstrel/command/RemoveSongCommand.java
index 935dc29..4a202a2 100644
--- a/src/main/java/net/knarcraft/minstrel/command/RemoveSongCommand.java
+++ b/src/main/java/net/knarcraft/minstrel/command/RemoveSongCommand.java
@@ -35,10 +35,12 @@ public class RemoveSongCommand implements CommandExecutor {
             return false;
         }
         if (index >= 0 && playlist.getSongs().size() > index) {
-            playlist.removeSong(index);
             //Stop any minstrels from playing the removed song
-            minstrelTrait.getPlaylist().stop();
-            minstrelTrait.getPlaylist().play(minstrelTrait);
+            playlist.stop();
+            playlist.removeSong(index);
+            if (!playlist.getSongs().isEmpty()) {
+                playlist.play(minstrelTrait);
+            }
             sender.sendMessage("Song removed");
         } else {
             sender.sendMessage("The specified index is outside the bounds of the playlist");
diff --git a/src/main/java/net/knarcraft/minstrel/music/Playlist.java b/src/main/java/net/knarcraft/minstrel/music/Playlist.java
index 1639749..3d21d6b 100644
--- a/src/main/java/net/knarcraft/minstrel/music/Playlist.java
+++ b/src/main/java/net/knarcraft/minstrel/music/Playlist.java
@@ -6,7 +6,9 @@ import org.bukkit.Bukkit;
 import org.bukkit.entity.Player;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 /**
  * A representation of a playlist containing songs
@@ -17,6 +19,7 @@ public class Playlist {
     private final boolean loop;
     private int currentlyPlaying = 0;
     private int schedulerId = -1;
+    private final Map<Player, Song> playerCurrentSong = new HashMap<>();
 
     /**
      * Instantiates a new playlist
@@ -62,10 +65,10 @@ public class Playlist {
      * @param player <p>The player to stop the playlist for</p>
      */
     public void stop(Player player) {
-        for (Song song : this.songs) {
-            if (song.isPlaying()) {
-                song.stop(player);
-            }
+        Song currentlyPlaying = playerCurrentSong.get(player);
+        if (currentlyPlaying != null) {
+            currentlyPlaying.stop(player);
+            playerCurrentSong.remove(player);
         }
     }
 
@@ -73,14 +76,14 @@ public class Playlist {
      * Stops all songs in this playlist for all players, and aborts scheduling for the next song
      */
     public void stop() {
-        for (Song song : this.songs) {
-            if (song.isPlaying()) {
-                for (Player player : Bukkit.getOnlinePlayers()) {
-                    song.stop(player);
-                }
+        for (Player player : Bukkit.getOnlinePlayers()) {
+            for (Song song : this.getSongs()) {
+                song.stop(player);
             }
         }
-        Bukkit.getScheduler().cancelTask(schedulerId);
+        if (Bukkit.getScheduler().isCurrentlyRunning(schedulerId)) {
+            Bukkit.getScheduler().cancelTask(schedulerId);
+        }
     }
 
     /**
@@ -100,6 +103,7 @@ public class Playlist {
 
         stop(player);
         Song currentSong = this.songs.get(this.currentlyPlaying - 1);
+        playerCurrentSong.put(player, currentSong);
         currentSong.play(trait, player, trait.getVolume(), trait.getPitch());
     }
 
@@ -122,12 +126,6 @@ public class Playlist {
             }
         }
 
-        for (Player player : Bukkit.getOnlinePlayers()) {
-            for (Song song : this.songs) {
-                song.stop(player);
-            }
-        }
-
         Song currentSong = this.songs.get(this.currentlyPlaying);
         currentSong.play(trait, trait.getVolume(), trait.getPitch());
         currentlyPlaying++;
@@ -136,7 +134,7 @@ public class Playlist {
                 Bukkit.getScheduler().cancelTask(schedulerId);
                 play(trait);
             }
-        }, currentSong.getDuration(), 20);
+        }, currentSong.getDuration() * 20L, 20);
     }
 
     /**