I have a similar problem with unused (hidden) cross references imported from Word document.
I end up with this two scripts:
First removes unused CR all at once, and second which show position of CR and then ask you if you want to delete one or proceed to next one

// —- First One —-

Array.prototype.exists = function(search){
for (var i=0; i<this.length; i++)
if (this[i] == search) return true;
return false;
}

var allUsedSources = getHyperlinksUsedSources(); // ALL hyperlink sources
var allCRSources = app.activeDocument.crossReferenceSources.everyItem().getElements(); //All Cros Reference Sources
var counter = 0;

// Check if Cros Reference Source is used
for (var i = allCRSources.length-1;i >= 0; i–){
if (!allUsedSources.exists(allCRSources[i])){
allCRSources[i].remove();
counter++;
}
}

alert (“Deleted ” + counter + ” unused Cross reference sources”);

function getHyperlinksUsedSources(){
return app.activeDocument.hyperlinks.everyItem().source;
}

// — END —

//— Second One —-

Array.prototype.exists = function(search){
for (var i=0; i<this.length; i++)
if (this[i] == search) return true;
return false;
}

var allUsedSources = getHyperlinksUsedSources();
var allCRSources = app.activeDocument.crossReferenceSources.everyItem().getElements();
var counter = 0;

for (var i = allCRSources.length-1;i >= 0; i–){
var myBoolean = allUsedSources.exists(allCRSources[i]);
if (!allUsedSources.exists(allCRSources[i])){
allCRSources[i].showSource();
if (confirm (“Unused Cross reference source text:”+””+allCRSources[i].sourceText.contents + “”+””+”Delete Cross reference?”)){
allCRSources[i].remove();
counter++;
};
if (!confirm (“Continue?”)){
exit();
}
}
}

alert (“Deleted ” + counter + ” unused Cross reference sources”);

function getHyperlinksUsedSources(){
return app.activeDocument.hyperlinks.everyItem().source;
}

// — END —