Hello everybody,
AS usual, I just come to update the MACRO i created for the people using FEI Tecnai microscopes, with the Digitalmicrograph for TEM and TIA for STEM.
I am quite proud of what i managed to get on the last few months, and I think I will not be updating the macro anymore.
I made a resume of what it does:
DM3
IMAGE -> converts to tif (or other format as jpg) and adds adequate SCALEBAR.
ELE.DIFF-> converts to tif and ans adequate SCALEBAR (1/nm p.e.).
EELS -> creates 2-columns .txt file, and tif image with the profile
SER
IMAGE -> converts to tif and adds SCALEBAR
EDS -> creates 2-columns .txt file, and a tif image with the profile
LINE SCAN -> creates n-columns .txt file, and a tif image with the spectra as pages
MAPPING -> creates a tiff with a page for each energy (I removed the 3-energies stuff because it was painful).
EMI
Gets quantifications related to each SER file embedded on the EMI File. Writes them in .txt files
Gets all the metadata and writes it as xml files, each for each SER file embeded on the EMI File.
If an Image was obtained in HAADF (and saved from the SER1 file) then it creates an image for each SER file and writes on it WHERE the EDS was performed.
TODO: I did not figure it out how to extract the images and spectra directly from the EMI files (even though i know they are there) to solve the “failed spectra” problem when the spectrum was stopped before finishing.
The line scan does not create an Image with the line for the scanning, because the TIA does not create a SER file for the Image, so I cannot get it from the EMI.
More testing and so on. If anyone wants to, I can help 
/// This macro batch processes .dm3 images saved with Gatan software
// for Transmission Electron Microscope (TEM) imaging. It adds scale bar to each image
// and converts them from .dm3 to different types of image
// the diffractions obtain a scalebar with 1/nm , and the EELS are saved
// as crappy image for track, and 2-column txt file
// it also converts .ser files saved with TIA (FEI) to images, adds scalebar to them
// and for the EDS a crappy image is created and a text file with the data
//finally, it gets the data from EMI files, and produces an XML file with the metadata
//and, in case, uses the Image created by SER converter to write WHERE the EDS have been performed
// global author: Oliver Dieste, 21.01.2016, JRC-ITU (Karlsruhe)
// last modification: 30.01.2019 10:00
// I would love to call this version the 1.0, as it is already perfectly functional, and
//does not seem to crash (too often).
//REQUIRES CSI DM3 Reader plugin http://code.google.com/p/cornell-spectrum-imager/wiki/Home
//for the processing of EELS files and EDS
// it also REQUIRES TIA Reader. I know, it is weird to need CSI READER and TIA READER
// but this is because images are better managed by TIA READER, while
//spectra must be managed by CSI TIA READER
// partially copied from: Amirhossein Khalajhedayati, 12/10/2013, University of California Irvine, USA
// Some other parts of this code were obtained from multiple resources on the web.
//in principle we do not want to deal with old versions of imagej. Just get a new version :)
requires("1.45s");
//choose directory
dir = getDirectory("Choose a directory that contains the files");
// now create a question form to ask
// overwrite images? convert DM3? convert SER ? EMI?
datos = newArray("", "", "", "", "", "");
//this is a function (see at the end of the file)
datos = getPreguntas();
format = datos[0];
bitdepth = datos[1];
overwrite = datos[2];
dm3 = datos[3];
ser = datos[4];
emi = datos[5];
start = getTime();
setBatchMode(true);
totaldm3 = 0;
totalser = 0;
totalemi = 0;
listdm3 = newArray(0);
listser = newArray(0);
listEMI = newArray(0);
setOption("ExpandableArrays", true);
totalfailed = 0;
totalskipped = 0;
espazo = " ";
var quantification = "";
//clean the log
print("\\Clear");
//start the process: lets find the images in the folder and SUBFOLDERS
find(dir);
//print a count of the time
print("######END######");
print("Processing time: " + (getTime() - start) / 1000 + " Seconds");
print("SER files converted: " + totalser);
print("EMI files converted: " + totalemi);
print("DM3 files converted: " + totaldm3);
print(" files failed: " + totalfailed);
print(" files skipped: " + totalskipped);
//END
//#####################################################//
function find(dir) {
// this is the function that searches for files inside folder and subfolders
//showStatus("enter " + dir);
//this function iterates in the sub-folders to find the dm3 files
// copied from the ListFilesRecursively macro
list = getFileList(dir);
for (i = 0; i < list.length; i++) {
if (endsWith(list[i], "/")) {
//if you find a dir, go deep on it
find("" + dir + list[i]);
} else if (endsWith(list[i], "dm3") && dm3) {
listdm3 = Array.concat(listdm3, dir + list[i]);
//if it is a dm3, add it to the list
} else if (endsWith(list[i], "ser") && ser) {
listser = Array.concat(listser, dir + list[i]);
//if it is a ser, add it to the list
} else if (endsWith(list[i], "emi") && emi) {
listEMI = Array.concat(listEMI, dir + list[i]);
//if it is a emi, add it to the list
}
}
//now we go through the lists to convert the files in order
for (i = 1; i < listdm3.length; i++) {
//first for DM3
figure = replace(listdm3[i], ".dm3", "." + format);
//we check if the image is already there. If we dont want to overwrite, we skip it.
if ((File.exists(figure) && overwrite=="No")||(!File.exists(figure) && overwrite=="Only update")) {
print(espazo + "SKIPPED DM3 " + figure);
totalskipped++;
} else{
convert_dm3(listdm3[i]);
}
showProgress(i, listdm3.length + listser.length + listEMI.length);
//this part is to be able to stop the macro at any time pressing BAR
interruptMacro = isKeyDown("space");
if (interruptMacro == true) {
print("interrupted");
setKeyDown("none");
break;
}
}
for (i = 1; i < listser.length; i++) {
//now for SER
//we check if the image is already there. If we dont want to overwrite, we skip it.
figure = replace(listser[i], ".ser", "." + format);
if ((File.exists(figure) && overwrite=="No")||(!File.exists(figure) && overwrite=="Only update")) {
print(espazo + "SKIPPED SER" + figure);
totalskipped++;
} else {
convert_ser(listser[i]);
}
//we try to make the progress bar as big as the number of total images, and move on it
showProgress(
i + listdm3.length,
listdm3.length + listser.length + listEMI.length
);
}
for (i = 1; i < listEMI.length; i++) {
//now for SER
//we check if the image is already there. If we dont want to overwrite, we skip it.
figure = replace(listEMI[i], ".emi", "." + format);
if ((File.exists(figure) && overwrite=="No")||(!File.exists(figure) && overwrite=="Only update")) {
print(espazo + "SKIPPED EMI" + figure);
totalskipped++;
} else {
convert_emi(listEMI[i]);
}
showProgress(
listdm3.length + listser.length + i,
listdm3.length + listser.length + listEMI.length
);
}
}
//#####################################################//
function convert_emi(file_name) {
imagestart = getTime();
//if we find a emi file, we open it as text
resto = File.openAsString(file_name);
if (resto != "") {
//now we go for the quantification
nome = split(file_name, "\\/");
trimm = replace(nome[nome.length - 1], ".emi", "");
restoxml = resto;
//trimm the raw data from starting to Normalp , that it is where the image starts
resto = getSubstring(resto, "0", "Normalp");
//we will save a text file for each quantification found
//with the name of this file. This is done because all the LOG file in the TIA
//software is saved on each emi file, unless you clear it out every time
starting = "= = = Document: " + trimm;
ending = "= = = E D X Q u a n t i f i c a t i o n R e s u l t s = = =";
i = 0;
do {
//this bucle searches for "start", then for "end", and saves the text between them as quantification
desde = indexOf(resto, starting);
if (desde > 0) {
ata = indexOf(resto, ending, desde);
if (ata < desde) {
quantification = substring(resto, desde);
resto = "";
} else {
quantification = substring(resto, desde, ata);
resto = substring(resto, ata);
}
if (lengthOf(quantification) > 500) {
//here we save it
File.saveString(
quantification,
replace(file_name, ".emi", "Q-" + i + ".txt")
);
}
i++;
} else {
resto = "";
}
} while (resto != "");
//when the resto is empty, there is nothing else to read
//repeat for xml. We will get the XML file with the METADATA
//again, it appears for each SER saved, so we will have several usually
i = 0;
j = 0;
starting = "<ObjectInfo>";
ending = "</ObjectInfo>";
do {
desde = indexOf(restoxml, starting);
if (desde > 0) {
ata = indexOf(restoxml, ending, desde);
if (ata < desde) {
quantification = "";
restoxml = "";
} else {
quantification = replace(
replace(getSubstring(restoxml, starting, ending), ">", ">"),
"<",
"<"
);
restoxml = substring(restoxml, ata);
}
if (lengthOf(quantification) > 500) {
j++;
File.saveString(
starting + quantification + ending,
replace(file_name, ".emi", "_" + j + ".xml")
);
}
//inside the metadata we may find position or area where EDS was done. Check it
position = getSubstring(
quantification,
"<ScanArea>(",
") um</ScanArea>"
);
beam = getSubstring(
quantification,
"<BeamPosition>(",
") um</BeamPosition>"
);
//we will just get the first image of the series to write on it
figure = replace(file_name, ".emi", "_1.tif");
if (File.exists(figure) && (position > "" || beam > "")) {
open(figure);
//if we have an image to write in, we will put a square and "EDS X" on it
//there may be a "position" or "area". We want to draw a square there
if (position > "") {
sep = "\\) \\- \\(";
position = replace(position, sep, "\\$");
position = replace(position, ",", "\\$");
preposition = split(position, "\\$");
startx = parseFloat(preposition[0]);
starty = parseFloat(preposition[1]);
endx = parseFloat(preposition[2]);
endy = parseFloat(preposition[3]);
} else {
beam = replace(beam, ",", "\\$");
preposition = split(beam, "\\$");
startx = parseFloat(preposition[0]);
starty = parseFloat(preposition[1]);
endx = startx;
endy = starty;
}
width = getWidth;
getPixelSize(unit, pixelWidth, pixelHeight);
if (unit == "nm") {
pixelWidth = pixelWidth / 1000;
}
//ok, now we convert the positions into pixels, because until now they are in nm/um
startx = floor(startx / pixelWidth) + width / 2;
starty = width / 2 - floor(starty / pixelWidth) ;
endx = floor(endx / pixelWidth) + width / 2;
endy = width / 2 - floor(endy / pixelWidth);
//in case they are the same, we extend the end a little
if (endx - startx <= 0) {
temporal = endx;
endx = startx + 1;
startx = temporal;
}
if (endy - starty <= 0) {
temporal = endy;
endy = starty + 1;
starty = temporal;
}
//we want the EDS square to be colorful
run("RGB Color");
setForegroundColor(255, 0, 0);
//caferefully we choose a font size adapted to the image size. Also the linewidth
letra = (60 * width) / 2048;
linha = (10 * width) / 2048;
setFont("SansSerif", letra, " antialiased");
makeRectangle(startx, starty, endx - startx, endy - starty);
run("Properties... ", "name=eds stroke=red width=linha fill=none");
run("Draw");
//did not test this part: if we will write out of the image, just go back
if (endx + 4 * letra > width) {
endx = endx - 4 * letra;
}
if (endy + 4 * letra > width) {
endy = endy - 4 * letra;
}
//write the text and save it
makeText("EDS " + j, endx, endy);
run("Draw");
figure = replace(file_name, ".emi", "_" + j + "-EDS.tif");
save(figure);
close();
}
i++;
} else {
restoxml = "";
}
} while (restoxml != "" && i < 20);
//ready. Lets close this function and go for the next picture :)
print(espazo + "READY EMI " + file_name);
totalemi++;
} else {
print(espazo + "FAILED EMI" + file_name);
totalfailed++;
}
}
//#####################################################//
function convert_ser(file_name) {
//this function converts the image
path = file_name;
if (nImages > 0) {
close("*");
}
//partially borrowed from ser2tiff plugin
//by J Kaelber at National University of Singapore on 20 July 2012
IJ.redirectErrorMessages();
run("CSI TIA Reader", "load=[&path]");
//first check if the reader could open an image
if (nImages > 0) {
colheinfo = getImageInfo();
//checking carefully the info obtained from that command, we found out that some keywords appear
//only on certain type of files. Wew make use of it
if (indexOf(colheinfo, "Depth:") > 0) {
//if it is a map, we pretend to do something great, choosing energy to get maps
//but this is not yet great. Just ask the energies and save the mapping for each
//on a 2.0 version we could think on have a database of energies and just choose elements, but it was too long
// energia = newArray("", "", "");
// Dialog.create("Mapping");
// Dialog.addMessage(
// "there seems to be a Mapping. Choose energy windows for analyses"
// );
//
// Dialog.addNumber("energy1", 0.01, 2, 4, "keV");
// Dialog.addNumber("energy2", 0.01, 2, 4, "keV");
// Dialog.addNumber("energy3", 0.01, 2, 4, "keV");
// Dialog.show();
// energia[0] = 100*Dialog.getNumber(); energia[1] = 100*Dialog.getNumber(); energia[2] = 100*Dialog.getNumber();
// //for(i=0;i<=3;i++{energia[i]=energia[i]*100;}
// if(energia[0]<nSlices&&energia[1]<nSlices&&energia[2]<nSlices&&(energia[0]>0 ||energia[1]>0||energia[2]>0)){
// energies=" slices="+energia[0]+","+energia[1]+","+energia[2];
// //extract the images corresponding to those energies.
// run("Make Substack...", energies);
run(bitdepth+"-bit");
//this set scale are on the plots along the convert macro to allow Windows to understand the size of the tif figures.
run("Set Scale...", "distance=0 known=0 pixel=1 unit=pixel");
}
else if (indexOf(colheinfo, "Height: 1 pixels") > 0) {
// then it´s a EDS
run("Select All");
// saveAs("Text Image", path);
x = getProfile();
energy = newArray(x.length);
str = "Energy(eV)\tCounts\n"; // header
for (i = 0; i < x.length; i++) {
energy[i] = i/100;
str += "" + energy[i] + "\t" + x[i] + "\n";
}
File.saveString(str, replace(path, ".ser", "") + ".txt");
//just make it slightly nicer, with normal axis labels and so on
setFont("Serif", 12, "antialiased");
setColor("black");
setJustification("left");
Plot.create("EELS", "Energy(eV)", "Counts");
Plot.setLineWidth(2);
Plot.setColor("red");
Plot.add("line",energy,x);
Plot.setLimits(0, 20, 0, NaN);
Plot.setColor("black");
//Plot.addText(s,0.65,0.1);
Plot.update();
run("Set Scale...", "distance=0 known=0 pixel=1 unit=pixel");
} else if (indexOf(colheinfo, "Resolution") > 0) {
// it is a line profile
//we save it as eds, and thats all. What else could we do here?
//we just save it as stack (one tif image with several pages) and text file
//first we save it as text
run("Select All");
saveAs("Text Image", path);
//then we close it
// close();
//run("TIA Reader", ".ser-reader...=&path");
//run("Images to Stack", "name=[Line Profile] title=[] use");
height = getHeight;
width=getWidth;
id=getImageID();
for (i=0;i<height;i++){
makeRectangle(0,i,width,1);
x=getProfile();
Plot.create("EDS PROFILE-"+i, "Energy(keV)", "Counts");
Plot.setLineWidth(2);
Plot.setColor("red");
Plot.setLimits(0, 2000, 0, NaN);
Plot.add("line",x);
Plot.update();
selectImage(id);
}
close("*.ser");
run("Images to Stack", "name=Stack title=[] use");
//run("Set Scale...", "distance=0 known=0 pixel=1 unit=pixel");
}
else {
// then it is an image
close();
run("TIA Reader", ".ser-reader...=&path");
// this part is adapted from the source poposed by @author Frederick Ding
getPixelSize(unit, pw, ph);
// then it is an image
doble = pw;
width = getWidth;
height = getHeight;
//run("Set Scale...", "distance=1 known=&doble pixel=1 unit=&unit");
crea_bar(doble, width);
run(bitdepth+"-bit");
}
//just in case we want the image to be in 16bit. If not, set it to 8bit
saveAs(format, path);
print(espazo + "READY SER " + file_name);
totalser++;
close("*");
}
else {
//if there was no opened image, it means it failed.
print(espazo + "FAILED SER " + file_name);
totalfailed++;
}
}
//#####################################################//
function convert_dm3(file_name) {
//this function converts the image
imagestart = getTime();
//partially borrowed from ser2tiff plugin
//by J Kaelber at National University of Singapore on 20 July 2012
path = file_name;
open(path);
if (nImages > 0) {
// close();
//run("CSI DM3 Reader", "load=[&path]");
getPixelSize(unit, pixelWidth, pixelHeight);
//we check if it is EELS with the signal value
eels = getTag("ImageTags.Meta Data.Signal");
//if it is not EELS
if (eels != " EELS") {
// ********************************************************//
//now we check if image or diffraction
modo = getTag("ImageTags.Microscope Info.Imaging Mode");
if (modo != " IMAGING") {
//for diffraction change scale unit
run("Set Scale...", "unit=1/" + unit);
}
//for both images and diffraction create a scale bar
//and save the images
width = getWidth;
crea_bar(pixelWidth, width);
run(bitdepth+"-bit");
} else {
close();
run("CSI DM3 Reader", "load=[&path]");
// ********************************************************//
//if it is EELS
//spectrum needs to be centered and calibrated
//we get the drift of the tube, the offset energy and the dispersion
//from the values recorded on the file
drift = parseFloat(
getTag("ImageTags.EELS.Acquisition.Spectrometer.Drift tube voltage")
);
offset = parseFloat(
getTag("ImageTags.EELS.Acquisition.Spectrometer.Prism offset")
);
dispersion = parseFloat(
getTag("ImageTags.EELS.Acquisition.Spectrometer.Dispersion")
);
aperture = parseFloat(
getTag("ImageTags.EELS.Acquisition.Spectrometer.Aperture label")
);
magnification = parseFloat(
getTag("ImageTags.Microscope Info.Indicated Magnification")
);
exp_time = parseFloat(getTag("ImageTags.DataBar.Exposure Time"));
sum_spectra = parseFloat(
getTag("ImageTags.Acquisition.Parameters.Objects.0.Parameter 1")
);
// evilly copied from the macro of
// Version 1.0, 24-Sep-2010 Michael Schmid
//this part gets the profile, saves it in variables x and y, and save them in
//a 2-column txt file
run("Select All");
x = getProfile();
//run("Plot Profile");
//Plot.getValues(x, y);
energy = newArray(x.length);
//this is width of the profile, where it starts and where it ends.
energy_width = 2 * dispersion * x.length;
// TODO: check what is wrong with the spectra where OFFSET exist , p.e. for M4,5 lines
//when using -3500eV, sometimes you get up to 11 eV of difference
inicio = drift - offset - floor(energy_width / 10);
// this is just in case the spectrum starts in negative values, for the floor to be over, not under
if (inicio < 0) {
inicio = inicio + 1;
}
fin = inicio + energy_width;
str = "Energy(eV)\tCounts\n"; // header
for (i = 0; i < x.length; i++) {
energy[i] = inicio + 2 * dispersion * i;
str += "" + energy[i] + "\t" + x[i] + "\n";
}
File.saveString(str, replace(path, ".dm3", "") + ".txt");
s =
"Aperture: " +
aperture +
" mm" +
"\n" +
"Camera length: " +
magnification +
" mm" +
"\n" +
"Exposure time: " +
exp_time +
" s" +
"\n" +
"Number of spectra: " +
sum_spectra +
"\n" +
"Dispersion: " +
dispersion +
" ev/ch";
setFont("Serif", 12, "antialiased");
setColor("black");
setJustification("left");
Plot.create("EELS", "Energy(eV)", "Counts");
Plot.setLineWidth(2);
Plot.setColor("red");
Plot.add("line",energy,x);
Plot.setColor("black");
Plot.addText(s,0.65,0.1);
Plot.update();
run("Set Scale...", "distance=0 known=0 pixel=1 unit=pixel");
}
saveAs(format, path);
print(espazo + "READY DM3 " + file_name);
totaldm3++;
close();
} else {
print(espazo + "FAILED DM3 " + file_name);
totalfailed++;
}
}
//#####################################################//
function getSubstring(string, prefix, postfix) {
//this function has been stolen somewhere on the net, not sure where
//it gets a substring out of a string
start = indexOf(string, prefix) + lengthOf(prefix);
end = start + indexOf(substring(string, start), postfix);
if (start >= 0 && end > start) return substring(string, start, end);
else return "";
}
//#####################################################//
function getPreguntas() {
//choose the output format
formats = newArray(
"tif",
"jpg",
"gif",
"png",
"pgm",
"bmp",
"fits",
"txt",
"zip",
"raw"
);
bits = newArray("8", "16","24");
items = newArray("Yes", "No", "Only update");
questions = newArray("", "", "", "", "", "");
Dialog.create("questions");
Dialog.addChoice("Convert to: ", formats, "tif");
Dialog.addChoice("Bit depth: ", bits, "8");
Dialog.addRadioButtonGroup("Overwrite", items, 1, 3, "No");
Dialog.addCheckbox("Convert TEM/DIFF/EELS? ", true);
Dialog.addCheckbox("Convert STEM/EDS? ", true);
Dialog.addCheckbox("Retrieve data from EMI? ", true);
Dialog.show();
questions[0] = Dialog.getChoice();
questions[1] = Dialog.getChoice();
questions[2] = Dialog.getRadioButton();
questions[3] = Dialog.getCheckbox();
questions[4] = Dialog.getCheckbox();
questions[5] = Dialog.getCheckbox();
return questions;
}
//#####################################################//
function getTag(tag) {
//get the needed tag
info = getImageInfo();
index1 = indexOf(info, tag);
if (index1 == -1) return "";
index1 = indexOf(info, "=", index1);
if (index1 == -1) return "";
index2 = indexOf(info, "\n", index1);
value = substring(info, index1 + 1, index2);
return value;
}
//#####################################################//
function crea_bar(ancho, pixel) {
//function that prints the scalebar with certain width
//first calculate the ideal width
// it is a third of the size of one pixel times the number of pixels
baseWidth = (ancho * pixel) / 3;
scaleValues = newArray(1, 2, 3, 4, 5);
oldWidth = 1;
factor = 1;
j = 0;
for (t = 0; t < 20; t++) {
scaleWidth = scaleValues[j] * factor;
if (baseWidth > scaleWidth) {
j++;
oldWidth = scaleWidth;
if (j >= scaleValues.length) {
j = 0;
factor *= 10;
}
} else {
scaleWidth = oldWidth;
if (scaleWidth > baseWidth) {
scaleWidth = scaleWidth / 2;
}
t = 20;
}
}
//then print it
run(
"Scale Bar...",
"width=&scaleWidth height=8 font=50 color=White background=Black location=[Lower Right] bold"
);
return;
}
