How can I use JavaFX to play video on 3D shape?

Question

I am new to JavaFX and Java. I am wondering how to make a scene on a surface of a cube or any polyhedra? I would like to play video back on the surface of any 3D shape. How could I go about doing this?

Answer 1

Here is a sample which uses the in-built JavaFX MediaPlayer and takes periodic snapshots of a view on the media into a texture image mapped on to a 3D shape (in this case a Box). A rotation animation around the Y Axis is added so that the sides of the box may be seen in perspective.

import javafx.animation.*;
import javafx.application.*;
import javafx.geometry.Rectangle2D;
import javafx.scene.*;
import javafx.scene.image.WritableImage;
import javafx.scene.media.*;
import javafx.scene.paint.*;
import javafx.scene.shape.*;
import javafx.scene.transform.Rotate;
import javafx.stage.Stage;
import javafx.util.Duration;

// Display a rotating 3D box with a video projected onto its surface.
public class ThreeDMedia extends Application {

    private static final String MEDIA_URL =
            "http://download.oracle.com/otndocs/products/javafx/oow2010-2.flv";

    private static final int SCENE_W = 640;
    private static final int SCENE_H = 400;

    private static final double MEDIA_W = 540 * 2/3;
    private static final double MEDIA_H = 209 * 2/3;

    private static final Color INDIA_INK = Color.rgb(35, 39, 50);

    @Override
    public void start(Stage stage) {
        // create a 3D box shape on which to project the video.
        Box box = new Box(MEDIA_W, MEDIA_H, MEDIA_W);
        box.setTranslateX(SCENE_W / 2);
        box.setTranslateY(SCENE_H / 2);

        // create a media player for the video which loops the video forever.
        MediaPlayer player = new MediaPlayer(new Media(MEDIA_URL));
        player.setCycleCount(MediaPlayer.INDEFINITE);

        // create a media view for the video, sized to our specifications.
        MediaView mediaView = new MediaView(player);
        mediaView.setPreserveRatio(false);
        mediaView.setFitWidth(MEDIA_W);
        mediaView.setFitHeight(MEDIA_H);

        // project the video on to the 3D box.
        showMediaOnShape3D(box, mediaView);

        // rotate the box.
        rotateAroundYAxis(box);

        // create a point light source a fair way away so lighting is reasonably even.
        PointLight pointLight = new PointLight(
                Color.WHITE
        );
        pointLight.setTranslateX(SCENE_W / 2);
        pointLight.setTranslateY(SCENE_H / 2);
        pointLight.setTranslateZ(-SCENE_W * 5);

        // add a bit of ambient light to make the lighting more natural.
        AmbientLight ambientLight = new AmbientLight(
                Color.rgb(15, 15, 15)
        );

        // place the shape and associated lights in a group.
        Group group = new Group(
                box,
                pointLight,
                ambientLight
        );

        // create a 3D scene with a default perspective camera.
        Scene scene = new Scene(
                group,
                SCENE_W, SCENE_H, true, SceneAntialiasing.BALANCED
        );
        scene.setFill(INDIA_INK);
        PerspectiveCamera camera = new PerspectiveCamera();
        scene.setCamera(camera);

        stage.setScene(scene);
        stage.setResizable(false);

        // start playing the media, showing the scene once the media is ready to play.
        player.setOnReady(stage::show);
        player.setOnError(Platform::exit);
        player.play();
    }

       // Project video on to 3D shape.
    private void showMediaOnShape3D(Shape3D shape3D, final MediaView mediaView) {
        PhongMaterial material = new PhongMaterial();
        shape3D.setMaterial(material);

        Scene mediaScene = new Scene(
                new Group(mediaView),
                MEDIA_W, MEDIA_H
        );
        SnapshotParameters snapshotParameters = new SnapshotParameters();
        snapshotParameters.setViewport(
                new Rectangle2D(
                        0, 0, MEDIA_W, MEDIA_H
                )
        );
        WritableImage textureImage = mediaView.snapshot(
                snapshotParameters,
                null
        );
        material.setDiffuseMap(textureImage);

        AnimationTimer timer = new AnimationTimer() {
            @Override
            public void handle(long now) {
                mediaView.snapshot(
                        snapshotParameters,
                        textureImage
                );
            }
        };
        timer.start();
    }

    // Rotates a shape around the y axis indefinitely.
    private void rotateAroundYAxis(Shape3D shape3D) {
        RotateTransition rotateY = new RotateTransition(
                Duration.seconds(10),
                shape3D
        );

        rotateY.setAxis(Rotate.Y_AXIS);
        rotateY.setFromAngle(360);
        rotateY.setToAngle(0);
        rotateY.setCycleCount(RotateTransition.INDEFINITE);
        rotateY.setInterpolator(Interpolator.LINEAR);

        rotateY.play();
    }

    public static void main(String[] args) {
        launch(args);
    }
}

Answer 2

Actually, you can play video on any 3D shape.

There's an excelent project for video playing from @caprica called VLCJ: A Java framework for the VLC media player.

While the project is intended for rendering in AWT canvas, the author has done some tests to render it also in JavaFX Canvas.

Based on his JavaFX class, it's easy to render the buffer on a 3D shape instead of a 2D canvas node.

Settings

First, you need to install first VLC video player.

Then you need some dependencies: vlcj-3.6.0.jar, jna-3.5-2.jar & platform-3.5.2.jar and slfj4j-api.1.7.12.jar.

Also, I'll use some custom 3D shapes from the FXyz library, though you can use the regular ones from the API, like a Box.

Basis

The trick to render video on a 3D shape is using the diffuse map of its material, that takes an image, and defines its texture.

You can find some more information on this here or here.

So for every frame available we will create a new image and set it as the diffuse map:

ByteBuffer byteBuffer = nativeBuffer.getByteBuffer(0, nativeBuffer.size());
BufferFormat bufferFormat = ((DefaultDirectMediaPlayer) mediaPlayerComponent.getMediaPlayer()).getBufferFormat();
WritableImage textureImage = new WritableImage(bufferFormat.getWidth(), bufferFormat.getHeight());
if (bufferFormat.getWidth() > 0 && bufferFormat.getHeight() > 0) {
    textureImage.getPixelWriter().setPixels(0, 0, bufferFormat.getWidth(), bufferFormat.getHeight(), pixelFormat, byteBuffer, bufferFormat.getPitches()[0]);
    // apply new frame as texture image to the 3D shape's material
    material.setDiffuseMap(textureImage);
}

An AnimationTimer will allow updating the frames and the texture.

Sample

This is a working sample that renders video on a segmented torus.

public class Video3D extends Application {

    static {
        // path to the VLC video player
        System.setProperty("jna.library.path", "C:/Program Files/VideoLAN/VLC");
    }

    // http://download.blender.org/peach/bigbuckbunny_movies/
    // (c) copyright 2008, Blender Foundation / www.bigbuckbunny.org
    private static final String VIDEO_FILE = "C:\\BigBuckBunny_320x180.mp4";

    private final DirectMediaPlayerComponent mediaPlayerComponent;
    private final WritablePixelFormat<ByteBuffer> pixelFormat;

    private final SegmentedTorusMesh torus = new SegmentedTorusMesh(50,40,12,3.2d,4.5d);
    private final PhongMaterial material = new PhongMaterial(Color.WHEAT);

    private double mousePosX, mousePosY;
    private double mouseOldX, mouseOldY;
    private final Rotate rotateX = new Rotate(-20, Rotate.X_AXIS);
    private final Rotate rotateY = new Rotate(240, Rotate.Y_AXIS);

    private final AnimationTimer timer;

    public TestVLC(){
        mediaPlayerComponent = new TestMediaPlayerComponent();
        pixelFormat = PixelFormat.getByteBgraInstance();
        timer = new AnimationTimer() {
            @Override
            public void handle(long now) {
                renderFrame();
            }
        };
    }
    protected void startTimer() {
        mediaPlayerComponent.getMediaPlayer().playMedia(VIDEO_FILE);
        timer.start();
    }

    protected void stopTimer() {
        mediaPlayerComponent.getMediaPlayer().stop();
        timer.stop();
    }

    @Override
    public void start(Stage primaryStage) {
        torus.setCullFace(CullFace.NONE);
        torus.setzOffset(1.4);
        torus.setMaterial(material);

        PerspectiveCamera camera = new PerspectiveCamera(true);
        camera.getTransforms().addAll (rotateX, rotateY, new Translate(0, 0, -30));

        Group root3D = new Group(camera,torus);

        SubScene subScene = new SubScene(root3D, 800, 600, true, SceneAntialiasing.BALANCED);
        subScene.setFill(Color.AQUAMARINE);
        subScene.setCamera(camera);

        BorderPane pane = new BorderPane();
        pane.setCenter(subScene);
        Button play = new Button("Play");
        play.setOnAction(e->startTimer());
        Button stop = new Button("Stop");
        stop.setOnAction(e->stopTimer());
        ToolBar toolBar = new ToolBar(play, stop);
        toolBar.setOrientation(Orientation.VERTICAL);
        pane.setRight(toolBar);
        pane.setPrefSize(600,400);

        Scene scene = new Scene(pane);

        scene.setOnMousePressed((MouseEvent me) -> {
            mouseOldX = me.getSceneX();
            mouseOldY = me.getSceneY();
        });
        scene.setOnMouseDragged((MouseEvent me) -> {
            mousePosX = me.getSceneX();
            mousePosY = me.getSceneY();
            rotateX.setAngle(rotateX.getAngle()-(mousePosY - mouseOldY));
            rotateY.setAngle(rotateY.getAngle()+(mousePosX - mouseOldX));
            mouseOldX = mousePosX;
            mouseOldY = mousePosY;
        });

        primaryStage.setScene(scene);
        primaryStage.setTitle("Video - JavaFX 3D");
        primaryStage.show();

    }

    @Override
    public final void stop() throws Exception {
        stopTimer();

        mediaPlayerComponent.getMediaPlayer().stop();
        mediaPlayerComponent.getMediaPlayer().release();
    }

    /**
     * Implementation of a direct rendering media player component that renders
     * the video to a JavaFX canvas.
     * https://github.com/caprica/vlcj-javafx/blob/master/src/test/java/uk/co/caprica/vlcj/javafx/test/JavaFXDirectRenderingTest.java
     */
    private class TestMediaPlayerComponent extends DirectMediaPlayerComponent {

        public TestMediaPlayerComponent() {
            super(new TestBufferFormatCallback());
        }
    }

    /**
     * Callback to get the buffer format to use for video playback.
     */
    private class TestBufferFormatCallback implements BufferFormatCallback {

        @Override
        public BufferFormat getBufferFormat(int sourceWidth, int sourceHeight) {
            final int width = sourceWidth;
            final int height = sourceHeight;
            Platform.runLater(() -> {
                torus.setMajorRadius(width/100);
                torus.setMinorRadius(height/40);
            });
            return new RV32BufferFormat(width, height);
        }
    }

    protected final void renderFrame() {
        Memory[] nativeBuffers = mediaPlayerComponent.getMediaPlayer().lock();
        if (nativeBuffers != null) {
            Memory nativeBuffer = nativeBuffers[0];
            if (nativeBuffer != null) {
                ByteBuffer byteBuffer = nativeBuffer.getByteBuffer(0, nativeBuffer.size());
                BufferFormat bufferFormat = ((DefaultDirectMediaPlayer) mediaPlayerComponent.getMediaPlayer()).getBufferFormat();
                WritableImage textureImage = new WritableImage(bufferFormat.getWidth(), bufferFormat.getHeight());
                if (bufferFormat.getWidth() > 0 && bufferFormat.getHeight() > 0) {
                    textureImage.getPixelWriter().setPixels(0, 0, bufferFormat.getWidth(), bufferFormat.getHeight(), pixelFormat, byteBuffer, bufferFormat.getPitches()[0]);
                    material.setDiffuseMap(textureImage);
                }
            }
        }
        mediaPlayerComponent.getMediaPlayer().unlock();
    }

    public static void main(String[] args) {
        launch(args);
    }

}

And these are just two snapshots of what you will get.