//TO-DO // add a cvs recorder for blop positions //timer import gab.opencv.*; import java.awt.Rectangle; import processing.video.*; import controlP5.*; import oscP5.*; import netP5.*; final static int OSC_IN_PORT = 8444; OscP5 mOscP5; OpenCV opencv; Capture video; PImage src, preProcessedImage, processedImage, contoursImage; //record time-lapse boolean record = false; boolean invert = true; ArrayList contours; // List of detected contours parsed as blobs (every frame) ArrayList newBlobContours; // List of my blob objects (persistent) ArrayList blobList; // Number of blobs detected over all time. Used to set IDs. int blobCount = 0; float scalefactor = 1.0, scaleX = 1.0, scaleY = 1.0 ; float contrast = 3.0; int brightness = 0; int threshold = 75; boolean useAdaptiveThreshold = false; // use basic thresholding boolean RecordTimelapse = false; // genera una imagen por segundo boolean screenmode = true; boolean blobSwitch = true; boolean bSwitch = true; boolean srcswitch = true; boolean helpflag = false; int thresholdBlockSize = 489; int thresholdConstant = 45; int blobSizeThreshold = 20; int blurSize = 4; IntList bAreas; IntList bX; IntList bY; int xID=0; int xArea=0; // Control vars ControlP5 cp5; int buttonColor; int buttonBgColor; long lastOscMillis; Textlabel myTitle; Textlabel controllers; boolean showcontrol = false; // Console vars String[] cons = {"1","2","3","4","5","6","7","8"}; boolean console = false; int consindex = 0; // camera vars String[] cameras = Capture.list(); int capW = 960, capH = 540; void setup() { // size(1160, 540); fullScreen(); println("screen width: " + width + "screen height: " + height); surface.setResizable(true); noStroke(); // clear console cons(1,"clear"); // list cameras printCameraList(); // frameRate(15); lastOscMillis = millis(); mOscP5 = new OscP5(this, OSC_IN_PORT); // init video capture size and start video setCaptureSize(16); // argument is the number of the camera (see camera list) // init Contours contours = new ArrayList(); // Blobs list blobList = new ArrayList(); // Init Controls cp5 = new ControlP5(this); initControls(); // Set thresholding toggleAdaptiveThreshold(useAdaptiveThreshold); // Reset Blob Timer btimer_reset(); bAreas = new IntList(); bX = new IntList(); bY = new IntList(); } //////////////////////////////////// ///////// DRAW LOOP //////////////// //////////////////////////////////// void draw() { background(0,0,0); // Read last captured frame if (video.available()) { video.read(); } // Load the new frame of our camera in to OpenCV opencv.useColor(); opencv.loadImage(video); // Flips the image horizontally opencv.flip(OpenCV.HORIZONTAL); src = opencv.getSnapshot(); /////////////////////////////// // <1> PRE-PROCESS IMAGE // - Grey channel // - Brightness / Contrast /////////////////////////////// // Gray channel opencv.gray(); //opencv.brightness(brightness); opencv.contrast(contrast); // Save snapshot for display preProcessedImage = opencv.getSnapshot(); /////////////////////////////// // <2> PROCESS IMAGE // - Threshold // - Noise Supression /////////////////////////////// // Adaptive threshold - Good when non-uniform illumination if (useAdaptiveThreshold) { // Block size must be odd and greater than 3 if (thresholdBlockSize%2 == 0) thresholdBlockSize++; if (thresholdBlockSize < 3) thresholdBlockSize = 3; opencv.adaptiveThreshold(thresholdBlockSize, thresholdConstant); // Basic threshold - range [0, 255] } else { opencv.threshold(threshold); } if(invert){ // Invert (black bg, white blobs) opencv.invert(); } // Reduce noise - Dilate and erode to close holes opencv.dilate(); opencv.erode(); // Blur opencv.blur(blurSize); // Save snapshot for display processedImage = opencv.getSnapshot(); /////////////////////////////// // <3> FIND CONTOURS /////////////////////////////// detectBlobs(); // Passing 'true' sorts them by descending area. //contours = opencv.findContours(true, true); // Save snapshot for display contoursImage = opencv.getSnapshot(); if (screenmode){ // Draw pushMatrix(); // Leave space for ControlP5 sliders //translate(width-src.width, 0); translate(0, 0); // Display images displayImages(); // Display contours in the lower right window pushMatrix(); //scale(0.5 * scalefactor); scale(0.5 * scaleX,0.5 * scaleY); translate(src.width, src.height * 1); // Contours if (bSwitch){displayContours();} //displayContoursBoundingBoxes(); // Blobs if (bSwitch){ displayBlobs();} popMatrix(); popMatrix(); } else{ // Draw pushMatrix(); // translate(width-src.width, 0); translate(0, 0); displayTrack(); // Display contours in the lower right window pushMatrix(); // scale(scalefactor); scale(scaleX, scaleY); // Contours if (bSwitch){displayContours();} //displayContoursBoundingBoxes(); // Blobs if (bSwitch){displayBlobs(); } popMatrix(); popMatrix(); }// end if screenmode // SHOW CONTROLS if(showcontrol){ // background controls fill(0, 80, 80, 200); noStroke(); rect(0, 0, 180, 270); showControls(); }else{hideControls();} // SHOW CONSOLE if(console){ showConsole(); } // SHOW HELP if(helpflag){ showHelp(); } // press R key for timelapse recording if (record == true) { saveFrame("timelapse-######.png"); delay(100); } if (record == false) { record = false; // Stop recording to the file } // write osc if (millis()-lastOscMillis > 100) { for (Blob b : blobList) { b.sendOsc(); } // lastOscMillis = millis(); } // draw blob lines and reset blobs every 3 seconds if(btimer_passed(3)){ resetBlobs(); btimer_reset(); } } // END DRAW LOOP /////////////////////// // Display Functions /////////////////////// void displayImages() { // pushMatrix(); // scale(0.5); // scale(scalefactor / 2.); scale(0.5 * scaleX,0.5 * scaleY); image(src, 0, 0); image(preProcessedImage, src.width, 0); image(processedImage, 0, src.height); image(src, src.width, src.height); popMatrix(); stroke(255); fill(255); textSize(12); text("Source", 10, 25); text("Pre-processed Image", src.width/2 * scalefactor + 10, 25); text("Processed Image", 10, src.height/2 * scalefactor + 25); text("Tracked Points", src.width/2 * scalefactor + 10, src.height/2 * scalefactor + 25); } //////////////////////// void displayTrack() { pushMatrix(); //scale(1.); // scale(scalefactor); scale(scaleX,scaleY); if(srcswitch){image(src, 0, 0);} popMatrix(); stroke(255); fill(255,0,0); textSize(24); if(srcswitch){text("Live Tracking", 10, 20);} } /////////////////////// void displayBlobs() { // if(blobSwitch){blobLines();} for (Blob b : blobList) { strokeWeight(1); b.display(); } } /////////////////////// void displayContours() { // Contours for (int i=0; i 0.9 * src.width * src.height) || (r.width < blobSizeThreshold || r.height < blobSizeThreshold)) continue; stroke(255, 0, 0); fill(255, 0, 0, 150); strokeWeight(2); rect(r.x, r.y, r.width, r.height); } } //////////////////// // Blob Detection //////////////////// void detectBlobs() { // Contours detected in this frame // Passing 'true' sorts them by descending area. contours = opencv.findContours(true, true); newBlobContours = getBlobsFromContours(contours); cons(1, "contours.size(): " + contours.size()); cons(2, "newBlobContours.size(): " + newBlobContours.size()); cons(3, "blobList.size(): " + blobList.size()); // Check if the detected blobs already exist are new or some has disappeared. if (blobSwitch){ // SCENARIO 1 // blobList is empty if (blobList.isEmpty()) { // Just make a Blob object for every face Rectangle for (int i = 0; i < newBlobContours.size (); i++) { // println("+++ New blob detected with ID: " + blobCount); blobList.add(new Blob(this, blobCount, newBlobContours.get(i))); blobCount++; /* // for bloblines Rectangle r = newBlobContours.get(i).getBoundingBox(); xArea=r.width * r.height; bAreas.append(xArea); int tempX = (int)r.getCenterX(); bX.append(tempX); int tempY=(int)r.getCenterY(); bY.append(tempX); */ } // SCENARIO 2 // We have fewer Blob objects than face Rectangles found from OpenCV in this frame } else if (blobList.size() <= newBlobContours.size()) { boolean[] used = new boolean[newBlobContours.size()]; // Match existing Blob objects with a Rectangle for (Blob b : blobList) { // Find the new blob newBlobContours.get(index) that is closest to blob b // set used[index] to true so that it can't be used twice float record = 50000; int index = -1; for (int i = 0; i < newBlobContours.size (); i++) { float d = dist(newBlobContours.get(i).getBoundingBox().x, newBlobContours.get(i).getBoundingBox().y, b.getBoundingBox().x, b.getBoundingBox().y); //float d = dist(blobs[i].x, blobs[i].y, b.r.x, b.r.y); if (d < record && !used[i]) { record = d; index = i; } } // Update Blob object location used[index] = true; b.update(newBlobContours.get(index)); } // Add any unused blobs for (int i = 0; i < newBlobContours.size (); i++) { if (!used[i]) { // println("+++ New blob detected with ID: " + blobCount); blobList.add(new Blob(this, blobCount, newBlobContours.get(i))); //blobList.add(new Blob(blobCount, blobs[i].x, blobs[i].y, blobs[i].width, blobs[i].height)); blobCount++; } } // SCENARIO 3 // We have more Blob objects than blob Rectangles found from OpenCV in this frame } else { // All Blob objects start out as available for (Blob b : blobList) { b.available = true; } // Match Rectangle with a Blob object for (int i = 0; i < newBlobContours.size (); i++) { // Find blob object closest to the newBlobContours.get(i) Contour // set available to false float record = 50000; int index = -1; for (int j = 0; j < blobList.size (); j++) { Blob b = blobList.get(j); float d = dist(newBlobContours.get(i).getBoundingBox().x, newBlobContours.get(i).getBoundingBox().y, b.getBoundingBox().x, b.getBoundingBox().y); //float d = dist(blobs[i].x, blobs[i].y, b.r.x, b.r.y); if (d < record && b.available) { record = d; index = j; } } // Update Blob object location Blob b = blobList.get(index); b.available = false; b.update(newBlobContours.get(i)); /* // for bloblines Rectangle r = newBlobContours.get(i).getBoundingBox(); xArea=r.width * r.height; bAreas.set(i,xArea); int tempX = (int)r.getCenterX(); bX.set(i,tempX); int tempY=(int)r.getCenterY(); bY.set(i,tempY); */ } // Start to kill any left over Blob objects for (Blob b : blobList) { if (b.available) { b.countDown(); if (b.dead()) { b.delete = true; } } } } // Delete any blob that should be deleted for (int i = blobList.size ()-1; i >= 0; i--) { Blob b = blobList.get(i); if (b.delete) { blobList.remove(i); } } } } /////////////////////// void resetBlobs(){ blobSwitch = false; for (int i = blobList.size ()-1; i >= 0; i--) { blobList.remove(i);} blobCount=0; for (int i=0; i0){ for(int i=1; i getBlobsFromContours(ArrayList newContours) { ArrayList newBlobs = new ArrayList(); // Which of these contours are blobs? for (int i=0; i 0.9 * src.width * src.height) || (r.width < blobSizeThreshold || r.height < blobSizeThreshold)) continue; newBlobs.add(contour); } return newBlobs; } ////////////////////////// // CONTROL P5 Functions ////////////////////////// void initControls() { cp5.addScrollableList("cameraz") .setPosition(180, 0) .setSize(300, 300) .setBarHeight(20) .setItemHeight(20) .addItems(cameras) .setCaptionLabel("video input") // .setType(ScrollableList.LIST) // currently supported DROPDOWN and LIST ; cp5.get(ScrollableList.class, "cameraz").setType(ControlP5.DROPDOWN); cp5.get(ScrollableList.class, "cameraz").close(); cp5.get(ScrollableList.class, "cameraz").setColorBackground(#008080); CColor c = new CColor(); c.setBackground(color(0,80,80)); c.setAlpha(100); cp5.get(ScrollableList.class, "cameraz").setColor(c); PFont pfont = createFont("Arial",20,true); // use true/false for smooth/no-smooth ControlFont font = new ControlFont(pfont,12); //label myTitle = cp5.addTextlabel("label") .setText("M I K R O T R A C K E R") .setPosition(16, 40) .setColorValue(color(0,0,0)) .setFont(font) //.setColorValue(0x000000ff) ; // Slider for contrast cp5.addSlider("contrast") .setLabel("contrast") .setPosition(10, 70) .setRange(0.0, 6.0) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for threshold cp5.addSlider("threshold") .setLabel("threshold") .setPosition(10, 90) .setRange(0, 255) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Toggle to activae adaptive threshold cp5.addToggle("toggleAdaptiveThreshold") .setLabel("use adaptive threshold") .setSize(10, 10) .setPosition(10, 120) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for adaptive threshold block size cp5.addSlider("thresholdBlockSize") .setLabel("a.t. block size") .setPosition(10, 150) .setRange(1, 700) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for adaptive threshold constant cp5.addSlider("thresholdConstant") .setLabel("a.t. constant") .setPosition(10, 170) .setRange(-100, 100) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for blur size cp5.addSlider("blurSize") .setLabel("blur size") .setPosition(10, 210) .setRange(1, 20) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for minimum blob size cp5.addSlider("blobSizeThreshold") .setLabel("min blob size") .setPosition(10, 230) .setRange(0, 60) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; /* // Toggle to activae record CVS file cp5.addToggle("toggleRecordTimelapse") .setLabel("record timelapse") .setSize(10, 10) .setPosition(15, 380) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; // Slider for timelapse cp5.addSlider("timelapse") .setLabel("time-lapse") .setPosition(15, 410) .setRange(0.01, 60) .setColorValue(color(255)) .setColorActive(color(0,100,0)) .setColorForeground(color(0,100,0)) .setColorBackground(color(0, 0, 0)) ; */ // Store the default background color, we gonna need it later buttonColor = cp5.getController("contrast").getColor().getForeground(); buttonBgColor = cp5.getController("contrast").getColor().getBackground(); } ////////////////////////// void toggleAdaptiveThreshold(boolean theFlag) { useAdaptiveThreshold = theFlag; if (useAdaptiveThreshold) { // Lock basic threshold setLock(cp5.getController("threshold"), true); // Unlock adaptive threshold setLock(cp5.getController("thresholdBlockSize"), false); setLock(cp5.getController("thresholdConstant"), false); } else { // Unlock basic threshold setLock(cp5.getController("threshold"), false); // Lock adaptive threshold setLock(cp5.getController("thresholdBlockSize"), true); setLock(cp5.getController("thresholdConstant"), true); } } ////////////////////////// void setLock(Controller theController, boolean theValue) { theController.setLock(theValue); if (theValue) { theController.setColorBackground(color(150, 150)); theController.setColorForeground(color(100, 100)); } else { theController.setColorBackground(color(buttonBgColor)); theController.setColorForeground(color(buttonColor)); } } ////////////////////////// void hideControls(){ cp5.getController("cameraz").hide(); cp5.getController("contrast").hide(); cp5.getController("label").hide(); cp5.getController("threshold").hide(); cp5.getController("toggleAdaptiveThreshold").hide(); cp5.getController("thresholdBlockSize").hide(); cp5.getController("thresholdConstant").hide(); cp5.getController("blurSize").hide(); cp5.getController("blobSizeThreshold").hide(); // cp5.getController("toggleRecordTimelapse").hide(); // cp5.getController("timelapse").hide(); } ////////////////////////// void showControls(){ cp5.getController("cameraz").show(); cp5.getController("contrast").show(); cp5.getController("label").show(); cp5.getController("threshold").show(); cp5.getController("toggleAdaptiveThreshold").show(); cp5.getController("thresholdBlockSize").show(); cp5.getController("thresholdConstant").show(); cp5.getController("blurSize").show(); cp5.getController("blobSizeThreshold").show(); // cp5.getController("toggleRecordTimelapse").show(); // cp5.getController("timelapse").show(); } ////////////////////////// void cameraz(int n) { /* request the selected item based on index n */ // println(n, cp5.get(ScrollableList.class, "cameraz").getItem(n)); cons(5, cameras[n]); cp5.get(ScrollableList.class, "cameraz").close(); video.stop(); setCaptureSize(n); // video = new Capture(this, capW, capH, cameras[n]); // video.start(); } ////////////////////////// void toggleRecordTimelapse(boolean theFlag) { RecordTimelapse = theFlag; if (RecordTimelapse) { // Lock basic threshold record = true; setLock(cp5.getController("timelapse"), true); } else { record = false; } } void keyPressed() { if (key == '0' ) { // toggle src image srcswitch = !(srcswitch); } if (key == '1') { // show capture / blob screens screenmode = !(screenmode); } if (key == 'C' || key == 'c') { // toggle showcontrol showcontrol = !(showcontrol); } if (key == 'I' || key == 'i') { // invert invert = !(invert); } if (key == '~' ) { // toggle console console = !(console); } if (key == 'B' || key == 'b') { // reset blobs resetBlobs(); } if (key == 'x') { bSwitch=!(bSwitch); } if (key == 's') { saveFrame("mikro-######.png"); } if (key == 'H' || key == 'h') { helpflag = !helpflag; } /* if (key == 'R' || key == 'r') { // Press R start time-lapse record = true; } if (key == 'S' || key == 's') { // Press s stop time-lapse record = false; } */ } /////// SHOW HELP MENU /////// void showHelp(){ fill(0, 0, 0, 90); noStroke(); int cleft = (width/2) - (width/4); int ctop = (height/2) - (height/4); rect(cleft,ctop, width/2, height/2); stroke(0,255,0); fill(0,255,0); textSize(24); text("key: h --> this help" , cleft+ 10, ctop + 30); text("key: c --> openCV controls" , cleft+ 10, ctop + 70); text("key: 1 --> Tracking full screen" , cleft+ 10, ctop + 110); text("key: 0 --> toggle source image (in full screen)" , cleft+ 10, ctop + 150); text("key: i --> invert fx image" , cleft+ 10, ctop + 190); text("key: s --> save frame" , cleft+ 10, ctop + 230); text("key: ~ --> message console" , cleft+ 10, ctop + 270); } ///////////////////////////////////////// // 8 LINES - BOTTOM SCREEN - CONSOLE ///////////////////////////////////////// void showConsole(){ fill(40, 40, 0, 200); noStroke(); int ctop=height-200; // height - height/4; rect(0,ctop, width, height); stroke(0,255,0); fill(0,255,0); textSize(14); int cs=cons.length - 1; text(cons[0] , 10, ctop + 20); text(cons[1] , 10, ctop + 40); text(cons[2] , 10, ctop + 60); text(cons[3] , 10, ctop + 80); text(cons[4] , 10, ctop + 100); text(cons[5] , 10, ctop + 120); text(cons[6] , 10, ctop + 140); text(cons[7] , 10, ctop + 160); } ////////////////////////////////////////// void cons(int cline, String mystring){ // consindex++; // consindex=consindex % 8; consindex = cline -1; cons[consindex]= "> " + mystring; // println(cons[consindex]); // clear if(cline==0 || mystring=="clear"){ for(int i=0; i<8;i++){ cons[i] = ">> "; } } } //////////////////////// void printCameraList(){ // print cameras list if (cameras.length == 0) { println("There are no cameras available for capture."); exit(); } else { println("Available cameras:"); for (int i = 0; i < cameras.length; i++) { println("index: " + i + " " + cameras[i]); } } } //////////////////////// void setCaptureSize(int n){ blobSwitch = false; String mystring = cameras[0]; int mybegin = mystring.indexOf("size=") + 5; int myend = mystring.indexOf(",fps"); // println("mybegin: " + mybegin + " myend: " + myend); String tempstring=mystring.substring(mybegin, myend); // println("tempstring: " + tempstring); int xbegin = tempstring.indexOf("x"); // println("x: " + xbegin); String stX=tempstring.substring(0, xbegin); String stY=tempstring.substring(xbegin+1,tempstring.length()); capW = int(stX); capH = int(stY); println("capW: " + capW + " capH: " + capH); cons(6,"capW: " + capW + " capH: " + capH); // capW = 960; // capH = 540; video = new Capture(this, capW, capH, cameras[n]); video.start(); scalefactor= float(width) / float(video.width); scaleX= float(width) / float(video.width); scaleY= float(height) / float(video.height); cons(8,"screen width: " + width + " video.width: " + video.width + " screen height: " + height + " video.height: " + video.height); cons(7,"scaleX: " + scaleX + " scaleY: " + scaleY); opencv = new OpenCV(this, capW, capH); blobSwitch = true; } ////////////////////////////// int btime = -1; void btimer_reset() { btime = millis(); } boolean btimer_passed(int seconds) { return ( millis() - btime > 1000 * seconds ); } //////////////////////////////