001 package util.sdss;
002 import java.io.*;
003 import java.util.*;
004
005 /** Extract quasars spectra and images from the SDSS DR1 using the preliminary
006 * identification flags. To restrict the results to quasars in final release catalog
007 * use the FinalQuasars filter which will read only quasars in the final quasar catalog.
008 * <ol>
009 * <li>Manually download SDSS from http://das.sdss.org/DR1/data/spectro/ss_tar_20/ </li>
010 * <li>Manually unzip as many as possible into a directory SDSS</li>
011 * <li>Manually Perform a recursive delete of spPlate-*.fits and plPlugMapM-*.par files</li>
012 * <li>Use this program to delete all spectra which don't have SPEC_QSO or SPEC_HIZ_QSO</li>
013
014 * Use the spDiag1d-MMMMM-FFFF.par file in each FFFF directory
015 */
016 public class ExtractQuasars {
017
018 public static void main(String[] args) {
019
020 // read in FinalQuasar catalog
021 // TODO:
022 List quasars = null;
023 try {
024 quasars = Quasar.readQuasars();
025 System.out.println(quasars.size() + " quasars read");
026 } catch (IOException e) {
027 System.out.println("Can't read quasar catalog : " + e);
028 System.exit(0);
029 }
030
031 // create a map between plate and fiber number and quasar as a check
032 Map<Integer, Quasar> map = new HashMap<Integer, Quasar>();
033 for (int i = 0; i < quasars.size(); i++) {
034 Quasar qso = (Quasar) quasars.get(i);
035 map.put(qso.getPrimaryKey(), qso);
036 }
037
038 File rootSDSSDir = new File("R:/SDSS");
039 if (args.length > 0) rootSDSSDir = new File(args[0]);
040
041 File[] plate = rootSDSSDir.listFiles();
042 for (int i = 0; i < plate.length; i++) {
043 //System.out.println(plate[i]);
044 int plateNum = Integer.parseInt(plate[i].getName());
045 File[] files = plate[i].listFiles(
046 new FilenameFilter() {
047 public boolean accept(File dir, String name) {
048 return (name != null && name.indexOf(".par") > -1);
049 }
050 });
051 File spSpec = new File(plate[i], "spSpec");
052 File spAtlas = new File(plate[i], "spAtlas");
053
054 if (files != null && files.length > 0) {
055 File parFile = files[0];
056 int julianDate = Integer.parseInt(parFile.getName().split("-")[1]);
057 try {
058 FileReader reader = new FileReader(parFile);
059 BufferedReader in = new BufferedReader(reader);
060 String row = "";
061 do {
062 row = in.readLine();
063 if (row != null && row.indexOf("DIAG ") > -1) {
064 int rbrakett = row.indexOf("{");
065 int fiber = Integer.parseInt(row.substring(5, rbrakett-1));
066 if (row.indexOf("QSO") == -1) { // Not a quasar
067
068 File spectraFile = Quasar.getSpectraFile(plateNum, fiber, julianDate);
069 File imageFile = Quasar.getImageFile(plateNum, fiber, julianDate);
070
071 if (!map.containsKey(Quasar.getPrimaryKey(plateNum, fiber))) {
072 if (spectraFile.delete()) System.out.println("deleted " + spectraFile);
073 if (imageFile.delete()) System.out.println("deleted " + imageFile);
074 } else {
075 //getSpectraImage(plateNum, fiber, julianDate);
076 //System.out.println("KEEP -- In DR1 final release quasar catalog :" + map.get(code));
077 }
078 } else {
079 //getSpectraImage(plateNum, fiber, julianDate);
080 }
081 }
082 } while (row != null);
083 reader.close();
084 } catch (IOException e) {
085 System.out.println("Can't read par file : " + e);
086 System.exit(0);
087 }
088 } else {
089 continue; //FIXME: if there is no PAR file look inside FITS files for SPEC_CLN header
090 }
091
092
093 // file.delete() all files in 'spAtlas' and 'spSpec' directories which
094 // have a fiber number which does not have 'QSO' somewhere on the line
095 // in 'spDiag...par' file
096
097 // TODO: get file
098
099 // Delete spectra and images if FFFF-fiber not in Final Catalog
100 // and does not have SPEC_QSO,SPEC_HIZ_QSO.
101 }
102 }
103
104 public static void getSpectraImage(int aPlate, int aFiber, int aJulianDate) {
105 //Get GIFS from http://das.sdss.org/DR1/data/spectro/1d_20/0266/gif/
106 java.net.URL gif = Quasar.getSpectraImageURL(aPlate, aFiber, aJulianDate);
107 System.out.println(gif);
108 // TODO: get
109 try {
110 InputStream in = gif.openStream();
111 String fileName = Quasar.getSpectraImageFileName(aPlate, aFiber, aJulianDate);
112 FileOutputStream out = new FileOutputStream(fileName);
113 byte[] buffer = new byte[40000];
114 while( true ) {
115 int bytesRead = in.read(buffer);
116 if (bytesRead == -1) {
117 break;
118 } else {
119 //System.out.println("first byte=" + buffer[0]);
120 out.write(buffer, 0, bytesRead);
121 }
122 }
123 out.close();
124 in.close();
125 } catch (IOException e ) {
126 System.err.println("Error downloading the GIF file " + e );
127 }
128
129 }
130
131 }