/* Last edited on 2011-08-22 17:22:50 by stolfilocal */ package main; import java.awt.BasicStroke; import java.awt.Color; import java.awt.Font; import java.awt.Graphics2D; import java.awt.image.BufferedImage; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; import javax.imageio.ImageIO; import association.HungarianAlgorithm; import association.Probabilities; import parameters.detection_arguments; import parameters.io_arguments; import parameters.tracking_arguments; import particlefiltering.particle; import particlefiltering.pfTracking; import preprocessing.Parser; import preprocessing.Util; import tracking.TrackRegion; import tracking.Pair; import tracking.TrackRelation; import classification.SVM; import classification.SVM_DATA; public class Strategy3 { /* Number of distinct text objects found so far: */ int num_text_objs = 0; public void Strategy ( ArrayList svm_descriptors, ArrayList svm_classifiers, io_arguments io_parameters, detection_arguments detection_parameters ) throws FileNotFoundException, ClassNotFoundException, IOException { File directory = new File (io_parameters.get_out_path() + String.format("%06d", 0)); directory.mkdir(); int bframe = io_parameters.get_bframe(); int eframe = io_parameters.get_eframe(); /* Get the HOG finger printing parameters: */ Parser parser = new Parser(); ArrayList HPAR = parser.Get_Array_List ("./inputs/hog_fingerprint.txt"); tracking_arguments TPAR = new tracking_arguments("./inputs/tracking_parameters.txt"); ArrayList particles_list = new ArrayList(); /* {T[i]} is the list of text regions detected in frame {bframe+i} */ ArrayList> T = new ArrayList>(); /* {P[i]} is the tracking relation between {T[i-1]} and {T[i]}. */ /* {P[0]} is always {null}. */ ArrayList> P = new ArrayList>(); /*Performing Text Detection at frame 0*/ BufferedImage frame0 = Get_Frame (bframe, io_parameters); detection_parameters = new detection_arguments(io_parameters.get_detection_parameters_file()); T.add(TextDetection.Detection (frame0, svm_descriptors, svm_classifiers, io_parameters, detection_parameters)); P.add(null); boolean detection = true; for (int iframe = bframe+1; iframe <= eframe; iframe++) { int i = iframe - bframe; /*Creating sub-directories (for each frame) to store the detection and tracking results*/ directory = new File (io_parameters.get_out_path() + String.format("%06d", iframe)); directory.mkdir(); /*Performing Text Detection at frame i*/ BufferedImage Vi = Get_Frame (iframe, io_parameters); BufferedImage Vj = Get_Frame (iframe-1, io_parameters); TrackRelation EX = Tracking ( Vj, Vi, T.get(i-1), particles_list, detection, TPAR, iframe); TrackRelation MX = null; if ((iframe % 30) == 0) { System.out.printf("Entrei na detecçao : %d\n", iframe); detection = true; detection_parameters = new detection_arguments(io_parameters.get_detection_parameters_file()); ArrayList Td = TextDetection.Detection (Vi, svm_descriptors, svm_classifiers, io_parameters, detection_parameters); MX = Merge (EX, Td, Vj, Vi, HPAR, iframe); } else { MX = EX; detection = false; } T.add(MX.R); P.add(MX.P); } Write_Detections (T, P, io_parameters); } /* Predicts the position of each region {e} from {Ti[0..T.size-1]} in the next frame, by linear extrapolation of its motion from the previous frame. Uses the tracking relation {Pi} to find the corresponding region in the previous frame. Returns {(L,R,P)} where {R} is the set of successfully extrapolated regions in the next frame, {L} are the corresponding regions in {Ti}, and {P} is the pairing between them. */ TrackRelation Extrapolate ( ArrayList Ti, ArrayList Pi, int wd, int ht) { TrackRelation EX = new TrackRelation(); for (int j = 0; j < Ti.size(); j++) { TrackRegion e = Ti.get(j); TrackRegion d = EX.ApplyInverseRelation (e, Pi); if (d != null) { TrackRegion f = Get_Extrapolate_Region (d, e, wd, ht); if (f != null) { EX.addPair(e,f); } } else { TrackRegion f = Copy_Track_Region (e); EX.addPair(e,f); } } return EX; } TrackRegion Copy_Track_Region (TrackRegion r) { TrackRegion n = new TrackRegion(); n.set_C(r.get_Cx(), r.get_Cy()); n.set_U(r.get_Ux(), r.get_Uy()); n.set_V(r.get_Vx(), r.get_Vy()); n.set_label(n.get_label()); return n; } /* Estimates the parameters of a text region {e} in the next frame,given its parameters {d} in the previous frame. Returns {null} is the extrapolated region lies outside the next frame, whhich is assumed to have {wd} columns by {ht} rows. */ TrackRegion Get_Extrapolate_Region (TrackRegion d, TrackRegion e, int wd, int ht) { TrackRegion f = new TrackRegion(); double[] Cd = d.get_C(); double[] Ud = d.get_U(); double[] Vd = d.get_V(); double[] Ce = e.get_C(); double[] Ue = e.get_U(); double[] Ve = e.get_V(); //System.out.printf("d = C : [%f,%f], U : [%f,%f], V : [%f,%f]\n", Cd[0], Cd[1], Ud[0], Ud[1], Vd[0], Vd[1]); //System.out.printf("e = C : [%f,%f], U : [%f,%f], V : [%f,%f]\n", Ce[0], Ce[1], Ue[0], Ue[1], Ve[0], Ve[1]); double[] Cf = Extrapolate_Position (Ce, Cd); double[] Uf = Extrapolate_Position (Ue, Ud); double[] Vf = Extrapolate_Position (Ve, Vd); f.set_C(Cf[0], Cf[1]); f.set_U(Uf[0], Uf[1]); f.set_V(Vf[0], Vf[1]); //System.out.printf("f = C : [%f,%f], U : [%f,%f], V : [%f,%f]\n\n\n", Cf[0], Cf[1], Uf[0], Uf[1], Vf[0], Vf[1]); f.set_label(e.get_label()); if (TextRegionInside(f, wd, ht)) { return f; } else { return null; } } boolean TextRegionInside(TrackRegion f, int wd, int ht) { for (int du = -1; du <= +1; du += 2) { for (int dv = -1; dv <= +1; dv += 2) { double px = f.get_Cx() + du*f.get_Ux() + dv*f.get_Vx(); double py = f.get_Cy() + du*f.get_Uy() + dv*f.get_Vy(); if ((px < 0) || (px > wd) || (py < 0) || (py > ht)) { return false; } } } return true; } double[] Extrapolate_Position (double[] p_2, double[] p_1) { assert ( (p_1.length == p_2.length) && (p_1.length == 2) ); double[] p = new double[2]; p[0] = 2*p_1[0] - p_2[0]; p[1] = 2*p_1[1] - p_2[1]; return p; } TrackRelation Merge ( TrackRelation EX, ArrayList D, BufferedImage Vj, BufferedImage Vi, ArrayList HPAR, int iframe ) { TrackRelation MX = new TrackRelation(); boolean transpose = false; double assignment_threshold = 0.05; /* Using Hungarian algorithm, to create the list with the best assignments */ double[][] SM = SimilarityMatrix (EX, D, Vj, Vi, HPAR, iframe); HungarianAlgorithm HA = new HungarianAlgorithm(); if (SM.length > SM[0].length) { SM = HA.transpose(SM); transpose = true; } /*Getting the list of assignments*/ int[][] A = HA.hgAlgorithm (SM, "max"); /*-------------------------------------------*/ /*Connect those elements in {E} that have an association in {D}*/ for (int i = 0; i < A.length; i++) { int ek = A[i][0]; int dk = A[i][1]; if (SM[ek][dk] > assignment_threshold) { TrackRegion Rd = null; /* Detected region. */ TrackRegion Re = null; /* Extrapolated region. */ if (transpose) { Rd = D.get(ek); Re = EX.R.get(dk); } else { Rd = D.get(dk); Re = EX.R.get(ek); } /* Keep the detected region {Rd}, paired to the antecedent of the extrapolated one {Re}: */ TrackRegion Rf = EX.ApplyInverseRelation (Re, EX.P); Rd.set_label(Rf.get_label()); MX.addPair(Rf, Rd); num_text_objs++; System.out.printf("ENTROU AQUI AQUI\n"); } } /* Insert those elements in {D} that do not have an association in {E} as new text objects: */ for (int i = 0; i < D.size(); i++) { TrackRegion Rd = D.get (i); if (! EX.TrackRegionBelong(Rd, MX.R)) { Rd.set_label(String.format("T:%03d", num_text_objs)); MX.addPair(null,Rd); num_text_objs++; } } return MX; } BufferedImage Get_Image_Region (BufferedImage image, TrackRegion R) { double[] C = R.get_C(); double[] U = R.get_U(); double[] V = R.get_V(); /*System.out.printf("C : %f %f\n", C[0], C[1]); System.out.printf("U : %f %f\n", U[0], U[1]); System.out.printf("V : %f %f\n", V[0], V[1]);*/ /* Handle axis-aligned rectangles only for now: */ assert(U[1] == 0); assert(V[0] == 0); int xmin = (int)(C[0] - U[0]); int xmax = (int)(C[0] + U[0]); int ymin = (int)(C[1] - V[1]); int ymax = (int)(C[1] + V[1]); //System.out.printf("xmin : %d, ymin : %d, xmax : %d, ymax : %d, width : %d, height : %d\n", xmin, ymin, xmax, ymax, image.getWidth(), image.getHeight()); BufferedImage region = Util.getSubImage(image, xmin, ymin, xmax, ymax); return region; } double[][] SimilarityMatrix ( TrackRelation EX, ArrayList D, BufferedImage Vj, BufferedImage Vi, ArrayList HPAR, int iframe) { ArrayList E = EX.R; double sigma_s = 10.0; double sigma_p = 10.0; double sigma_a = 5.0; Probabilities Pr = new Probabilities(); double[][] SM = new double[E.size()][D.size()]; Parser parser = new Parser(); PrintStream info = parser.Open_Print_Stream ("./out/" + String.format("%06d", iframe) + "/info.txt"); for (int ek = 0; ek < E.size(); ek++) { TrackRegion Re = E.get(ek); /* Extrapolated region in frame {Vi} */ TrackRegion Rf = EX.ApplyInverseRelation (Re, EX.P); /* Corresp. region in frame {Vj} */ BufferedImage Vf = Get_Image_Region (Vj, Rf); for (int dk = 0; dk < D.size(); dk++) { TrackRegion Rd = D.get (dk); BufferedImage Vd = Get_Image_Region (Vi, Rd); double prob_s = Pr.prob_s (Re, Rd, sigma_s); double prob_p = Pr.prob_p (Re, Rd, sigma_p); double prob_a = Pr.prob_a (Vf, Vd, sigma_a, HPAR); SM[ek][dk] = prob_s * prob_p * prob_a; info.printf("------------------------------------\n\n"); info.printf("Rd : C = [%f,%f], U = [%f,%f], V = [%f,%f], label : %s\n", Rd.get_Cx(), Rd.get_Cy(), Rd.get_Ux(), Rd.get_Uy(), Rd.get_Vx(), Rd.get_Vy(), Rd.get_label()); info.printf("Re : C = [%f,%f], U = [%f,%f], V = [%f,%f], label : %s\n", Re.get_Cx(), Re.get_Cy(), Re.get_Ux(), Re.get_Uy(), Re.get_Vx(), Re.get_Vy(), Re.get_label()); info.printf("Rf : C = [%f,%f], U = [%f,%f], V = [%f,%f], label : %s\n", Rf.get_Cx(), Rf.get_Cy(), Rf.get_Ux(), Rf.get_Uy(), Rf.get_Vx(), Rf.get_Vy(), Rf.get_label()); info.printf("P : %f, ps : %f, pp : %f, pa :%f\n", SM[ek][dk], prob_s, prob_p, prob_a); } } parser.Close_Print_Stream (info, ""); return SM; } void Write_Detections (ArrayList> T, ArrayList> P, io_arguments io_parameters) { int bframe = io_parameters.get_bframe(); int eframe = io_parameters.get_eframe(); for (int iframe = bframe; iframe <= eframe; iframe++) { int i = iframe - bframe; BufferedImage Vi = Get_Frame (iframe, io_parameters); write_set_regions (Vi, T.get(i), P.get(i), io_parameters, false, iframe); } } BufferedImage Get_Frame (int iframe, io_arguments io_parameters) { /*Getting the image frame name*/ String frame_name = io_parameters.get_frames_path() + String.format("%06d", iframe) + ".png"; /*Reading the image frame*/ try { BufferedImage frame = ImageIO.read(new File(frame_name)); return frame; } catch (Exception e) { e.printStackTrace(); System.exit(1); return null; } } void write_set_regions (BufferedImage frame, ArrayList set, ArrayList P_i_j, io_arguments io_parameters, boolean option, int iframe) { BufferedImage image = Util.imageScale (frame, 1.0); Graphics2D g = image.createGraphics(); g.setColor(Color.black); g.setStroke(new BasicStroke(2.0f)); Font f = new Font("SansSerif", Font.BOLD, 12); g.setFont(f); for (int box = 0; box < set.size(); box++) { TrackRegion A = set.get (box); String label = A.get_label(); double[] C = A.get_C(); double[] U = A.get_U(); double[] V = A.get_V(); int x = (int)(C[0] - U[0]); int y = (int)(C[1] - V[1]); int w = (int)(U[0]*2); int h = (int)(V[1]*2); g.drawRect(x, y, w, h); //g.drawString(label, x, y); DESCOMENTAR } g.dispose(); try { ImageIO.write(image, "png", new File(io_parameters.get_out_path() + String.format("%06d", iframe) + "/detection.png")); } catch (Exception e) { System.err.printf("cannot write image %s\n", io_parameters.get_out_path() + String.format("%06d", iframe) + "/detection.jpg"); } } TrackRelation Tracking ( BufferedImage Fj, BufferedImage Fi, ArrayList Tj, ArrayList particles_list, boolean add, tracking_arguments TPAR, int iframe) throws FileNotFoundException, ClassNotFoundException, IOException { pfTracking pf = new pfTracking(); TrackRelation EX = new TrackRelation(); if (add) { particles_list.clear(); particles_list = new ArrayList(); for (int j = 0; j < Tj.size(); j++) { TrackRegion Rj = Tj.get(j); double[] C = Rj.get_C(); double[] U = Rj.get_U(); double[] V = Rj.get_V(); int xmin = (int)(C[0] - U[0]); int xmax = (int)(C[0] + U[0]); int ymin = (int)(C[1] - V[1]); int ymax = (int)(C[1] + V[1]); int w = (int)(U[0]*2); int h = (int)(V[1]*2); BufferedImage Vj = Util.getSubImage(Fj, xmin, ymin, xmax, ymax); pf.Initialization(Vj, xmin, ymin, w, h, particles_list, j, TPAR); } } pf.Tracking (Fi, particles_list, particles_list.size(), iframe, TPAR); for (int i = 0; i < Tj.size(); i++) { TrackRegion Ri = new TrackRegion(); TrackRegion Rj = Tj.get(i); double s = particles_list.get(i)[0].get_s(); int w_r = Math.round((float)(particles_list.get(i)[0].get_y())); int w_c = Math.round((float)(particles_list.get(i)[0].get_x())); int w_w = Math.round((float)(particles_list.get(i)[0].get_width() * s)); int w_h = Math.round((float)(particles_list.get(i)[0].get_height() * s)); Ri.set_C (w_c, w_r); Ri.set_U (w_w/2, 0.0); Ri.set_V (0.0, w_h/2); if ( TextRegionInside (Ri, Fi.getWidth(), Fi.getHeight() ) ) { EX.addPair(Rj,Ri); } else { //VER TrackRegion f = Copy_Track_Region (Rj); EX.addPair(Rj,f); } } return EX; } }