FBO with stencil and AA

[ropelsw]

I need to render a scene offscreen with a resolution (w,h) different from the displayed one and save it as image on disk.
I tried using FBO: my code works, but I don't find a way to modify it adding a stencil buffer and using AA (multisample)... every attempt failed (I tried both adding a separate stencil buffer and using a 24/8 packed depth + stencil...)
Does anybody know how to modify it for stencil and AA ?

working code without stencil and AA:
int framebufferID = EXTFramebufferObject.glGenFramebuffersEXT();// create a new framebuffer
int colorTextureID = GL11.glGenTextures();// and a new texture used as a color buffer
int depthRenderBufferID = EXTFramebufferObject.glGenRenderbuffersEXT();// And finally a new depthbuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, framebufferID);// switch to the new framebuffer
GL11.glBindTexture(GL11.GL_TEXTURE_2D, colorTextureID);// Bind the colorbuffer texture
GL11.glTexParameterf(GL11.GL_TEXTURE_2D, GL11.GL_TEXTURE_MIN_FILTER, GL11.GL_LINEAR);// make it linear GL11.glTexImage2D(GL11.GL_TEXTURE_2D, 0, GL11.GL_RGBA8, width, height, 0,GL11.GL_RGBA, GL11.GL_INT, (ByteBuffer) null);
EXTFramebufferObject.glFramebufferTexture2DEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,GL11.GL_TEXTURE_2D, colorTextureID, 0); // attach it to the framebuffer
// initialize depth renderbuffer
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, depthRenderBufferID);// bind the depth renderbuffer
EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL14.GL_DEPTH_COMPONENT24, width, height);// get the data space for it
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_DEPTH_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depthRenderBufferID); // bind it to the renderbuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);
GL11.glBindTexture(GL11.GL_TEXTURE_2D, 0);
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, framebufferID);      // switch to rendering on our FBO
int framebuffer = EXTFramebufferObject.glCheckFramebufferStatusEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT);
switch (framebuffer) {
 ....
}
rendermyscene();
Display.update();
GL11.glReadBuffer(EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT);
GL11.glReadPixels(0, 0, width, height, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, buffer);
... save my buffer

[ropelsw]

Ok, I've finally got it to work: FBO + stencil + multisample, blitting to non multisample FBO to be able to get back and save the generated image data.
If anyone was interested, I can send the code or post it somewhere (some suggestions ?)

[kappa]

If the code is small enough (say a single class), then if you like you can post it on the lwjgl wiki and maybe write up a small tutorial for it.

[ropelsw]

I'm not enabled to post anything in the wiki, so here it is the code I'm using to generate an off-screen image using AA and stencil buffer:

Code:

boolean ok = false;
// We want FBO and Stencil. The only format really supported anywhere is Depth Buffer + Stencil (24 bit + 8 bit) packed in a single
// 32 bit buffer. Solutions with depth and stencil buffer separated seldom work
if (GLContext.getCapabilities().GL_EXT_framebuffer_object &&
GLContext.getCapabilities().GL_EXT_packed_depth_stencil
) {
System.out.println("Using frame buffer");
ok = true; // let's be optimist

int maxsamples = 1;
// enabling AA only if we have multisample capability and are able to blit the results (not readable as is) in a normal (not multisample) buffer
boolean multi = GLContext.getCapabilities().GL_EXT_framebuffer_multisample && GLContext.getCapabilities().GL_EXT_framebuffer_blit;
if(multi){
   // let's use as many samples we can
maxsamples = GL11.glGetInteger(EXTFramebufferMultisample.GL_MAX_SAMPLES_EXT);
System.out.println("Using multi" + maxsamples);
} else {
System.out.println("No multisamples (AA)");
}

int fbo_ID = EXTFramebufferObject.glGenFramebuffersEXT(); // we create a new framebuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID);// switch to the new framebuffer

// we allocate a depth buffer
int depth_ID = EXTFramebufferObject.glGenRenderbuffersEXT();
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID); // bind the depth renderbuffer
if(!multi){
   // format with packed stencil
EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, EXTPackedDepthStencil.GL_DEPTH_STENCIL_EXT, width, height); // get the data space for it
} else {
   // same as above, but multisample
EXTFramebufferMultisample.glRenderbufferStorageMultisampleEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, maxsamples,EXTPackedDepthStencil.GL_DEPTH_STENCIL_EXT, width, height); // get the data space for it
}
// we bind the same buffer twice: once as depth buffer, once as stencil
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_DEPTH_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID); // bind it to the renderbuffer
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_STENCIL_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID); // bind it to the renderbuffer

    // we bind the destination buffer
int color_ID = EXTFramebufferObject.glGenRenderbuffersEXT(); // and a new texture used as a color buffer
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, color_ID); // bind the depth renderbuffer
if(!multi){
EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL11.GL_RGBA, width, height); // get the data space for it
} else {
EXTFramebufferMultisample.glRenderbufferStorageMultisampleEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, maxsamples,GL11.GL_RGBA, width, height); // get the data space for it
}
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, color_ID); // bind it to the renderbuffer
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, 0); // bind the depth renderbuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);

// FBO render pass
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID); // switch to rendering on our FBO

// we'll check everithing is OK
int framebuffer = EXTFramebufferObject.glCheckFramebufferStatusEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT);
switch (framebuffer) {
case EXTFramebufferObject.GL_FRAMEBUFFER_COMPLETE_EXT:
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT_EXT");
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT_EXT");
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS_EXT");
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER_EXT");
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT");
break;
case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER_EXT:
ok = false;
System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER_EXT");
break;
default:
ok = false;
System.out.println("Unexpected reply from glCheckFramebufferStatusEXT: " + framebuffer);
break;
}
int sfbo_ID = 0;
int scolor_ID = 0;
if(ok) {
GL11.glViewport (0, 0, width, height);// set The Current Viewport to the fbo size

try {
GL11.glClear(GL11.GL_COLOR_BUFFER_BIT | GL11.GL_DEPTH_BUFFER_BIT | GL11.GL_STENCIL_BUFFER_BIT);

/*
      ... code where we do our rendering ...
*/

Display.update();

} catch (Exception e) {
e.printStackTrace();
}

// multibuffer isn't directly accessible: we have to blit the results in a single buffer FBO
if(multi){
   // we create the single buffer FBO
sfbo_ID = EXTFramebufferObject.glGenFramebuffersEXT();
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, sfbo_ID);
scolor_ID = EXTFramebufferObject.glGenRenderbuffersEXT();
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, scolor_ID);
EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL11.GL_RGBA, width, height); // get the data space for it
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, scolor_ID); // bind it to the renderbuffer
   // we set source and destination
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferBlit.GL_READ_FRAMEBUFFER_EXT, fbo_ID);
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferBlit.GL_DRAW_FRAMEBUFFER_EXT, sfbo_ID);
// ... and blit
EXTFramebufferBlit.glBlitFramebufferEXT(0, 0, width, height, 0, 0, width, height, GL11.GL_COLOR_BUFFER_BIT, GL11.GL_NEAREST);
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, sfbo_ID);
} else {
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID);
}
// whichever operation we did, now the buffer to be read is binded and we'll read it in a ByteBuffer
GL11.glReadBuffer(EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT);
ByteBuffer buffer = BufferUtils.createByteBuffer(width * height * 3);
GL11.glReadPixels(0, 0, width, height, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, buffer);
// we copy the readbuffer in a buffered image
BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
for(int x = 0; x < width; x++) {
for(int y = 0; y < height; y++) {
int i = (x + (width * y)) * 3;
int r = buffer.get(i) & 0xFF;
int g = buffer.get(i + 1) & 0xFF;
int b = buffer.get(i + 2) & 0xFF;
image.setRGB(x, height - (y + 1), (0xFF << 24) | (r << 16) | (g << 8) | b);
}
}

        // we write out our image, just to see it
try {
ImageIO.write(image, "JPG", new File("d:/temp/immagine.jpg"));
} catch (IOException e) {
e.printStackTrace();
}
   
}
// at the end we dispose our buffers...
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);
EXTFramebufferObject.glDeleteRenderbuffersEXT(depth_ID);
EXTFramebufferObject.glDeleteRenderbuffersEXT(color_ID);
EXTFramebufferObject.glDeleteFramebuffersEXT(fbo_ID);
if(ok && multi){
EXTFramebufferObject.glDeleteRenderbuffersEXT(scolor_ID);
EXTFramebufferObject.glDeleteFramebuffersEXT(sfbo_ID);
}
}