diff --git a/package.json b/package.json index 1a6fc0fa..f96a4842 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "react-intl": "^6.2.5", "react-router-dom": "^6.4.3", "react-sizeme": "^3.0.2", + "react-player": "^2.16.0", "sass": "^1.52.1", "semver": "^7.5.2", "tldraw-v1": "npm:@tldraw/tldraw@^1.2.7", diff --git a/src/components/external-video-player/index.js b/src/components/external-video-player/index.js new file mode 100644 index 00000000..c55a3844 --- /dev/null +++ b/src/components/external-video-player/index.js @@ -0,0 +1,409 @@ +import React, { PureComponent } from 'react'; +import ReactPlayer from 'react-player'; +import cx from 'classnames'; +import { defineMessages } from 'react-intl'; +import logger from 'utils/logger'; +//import { ID } from 'utils/constants'; +import { getCurrentDataIndex } from 'utils/data'; +import player from 'utils/player'; + +import './styles.css'; + +const intlMessages = defineMessages({ + autoPlayWarning: { + id: 'player.externalVideo.autoPlayWarning', + description: 'Shown when user needs to interact with player to make it work', + }, + +}); + + +const SYNC_INTERVAL_SECOND = 1; +const AUTO_PLAY_BLOCK_DETECTION_TIMEOUT_SECONDS = 5; +//const ORCHESTRATOR_INTERVAL_MILLISECOND = 300; +const IGNORE_STOP_CLOSE_TO_START_SECOND = 0.5; + +class ExternalVideoPlayer extends PureComponent { + + constructor(props) { + super(props); + + this.player = null; + + this.autoPlayTimeout = null; + + this.hasPlayedBefore = false; + this.playerIsReady = false; + + this.time = 0; + this.buffering= false; + this.lastTime = 0; + this.playerUpdateTime = -1; + this.primaryPlayerPlaying = false; + this.lastEventPlaybackRate = 1; + this.isOrchestrating = false; + this.rafId = null; + + this.state = { + muted: false, + playing: false, + autoPlayBlocked: false, + errorPlaying: false, + playbackRate: 1, + volume: 1, + urlPlayed: "", + }; + + this.opts = { + // default option for all players, can be overwritten + playerOptions: { + autoplay: false, + playsinline: true, + controls: false, + }, + file: { + attributes: { + controls: false, + autoPlay: false, + playsInline: true, + }, + }, + youtube: { + playerVars: { + autoplay: 0, + modestbranding: 1, + autohide: 1, + rel: 0, + ecver: 2, + controls: 1, + enablejsapi: 0, + showinfo: 0 + }, + }, + + preload: true, + }; + + + this.getCurrentTime = this.getCurrentTime.bind(this); + this.setPlaybackRate = this.setPlaybackRate.bind(this); + this.seekTo = this.seekTo.bind(this); + + this.handleFirstPlay = this.handleFirstPlay.bind(this); + this.handleOnReady = this.handleOnReady.bind(this); + this.handleOnPlay = this.handleOnPlay.bind(this); + this.handleOnPause = this.handleOnPause.bind(this); + this.handleVolumeChange = this.handleVolumeChange.bind(this); + this.handleOnBuffer = this.handleOnBuffer.bind(this); + this.handleOnBufferEnd = this.handleOnBufferEnd.bind(this); + + this.orchestrator = this.orchestrator.bind(this); + this.startOrchestrator = this.startOrchestrator.bind(this); + this.stopOrchestrator = this.stopOrchestrator.bind(this); + + this.autoPlayBlockDetected = this.autoPlayBlockDetected.bind(this); + + this.whichVideo = this.whichVideo.bind(this); + //this.dispatchTimeUpdate = this.dispatchTimeUpdate.bind(this); + } + + //dispatchTimeUpdate = (time) => { + // const event = new CustomEvent(EVENTS.TIME_UPDATE, { detail: { time }}); + // document.dispatchEvent(event); + //}; + + autoPlayBlockDetected() { + this.setState({ autoPlayBlocked: true }); + } + + handleFirstPlay() { + const { hasPlayedBefore } = this; + + if (!hasPlayedBefore) { + this.hasPlayedBefore = true; + + this.setState({ autoPlayBlocked: false }); + + if (this.autoPlayTimeout) { + clearTimeout(this.autoPlayTimeout); + } + + } + } + + getCurrentTime() { + const time = this.player?.getCurrentTime?.(); + return typeof time === 'number' ? time : 0; + } + + setPlaybackRate(value) { + //The original way to get the rate from props did not work, + // because props will not be updated after the initial rendering. + //const { primaryPlaybackRate } = this.props; + + // Rate depends on primary rate player + const rate = value * this.lastEventPlaybackRate; + + const currentRate = this.state.playbackRate; + + if (currentRate === rate) { + return; + } + + logger.debug(`external_video: setPlaybackRate current=${currentRate} primary=${value} lastEventPlaybackRate=${this.lastEventPlaybackRate} rate=${rate}`); + this.setState({ playbackRate: rate }); + + } + + handleOnReady() { + const { hasPlayedBefore, playerIsReady } = this; + + if (hasPlayedBefore || playerIsReady) { + return; + } + + this.playerIsReady = true; + this.handleFirstPlay(); + + //const { onPlayerReady } = this.props; + + //if (onPlayerReady) onPlayerReady(ID.EXTERNAL_VIDEOS, this); + + this.handleOnPlay(); + + } + + handleOnPlay() { + const { playing } = this.state; + + if (!playing && this.primaryPlayerPlaying) { + this.setState({ playing: true }); + this.handleFirstPlay(); + } + } + + handleOnPause() { + const { playing } = this.state; + + if (playing) { + this.setState({ playing: false }); + this.handleFirstPlay(); + } + } + + handleOnBuffer() { + this.buffering = true; + } + + handleOnBufferEnd() { + this.buffering = false; + } + + handleVolumeChange = (value, isMuted) => { + if (this.state.volume !== value ) { + this.setState({ volume: parseFloat(value)}); + logger.debug(`external_video: VolumeChange CV=${this.state.volume.toFixed(2)} NV=${value.toFixed(2)}`); + } + if (this.state.muted !== isMuted) { + this.setState({ muted: isMuted}); + logger.debug(`external_video: muteChange CM=${this.state.muted} NM=${isMuted}`); + } + } + + + seekTo(time) { + const { player } = this; + + if (!player) { + //return logger.error('No player on seek'); + return; + } + + // Seek if viewer has drifted too far away from presenter + if (Math.abs(this.getCurrentTime() - time) > SYNC_INTERVAL_SECOND) { + logger.debug(`Video synchronised! ${(time - this.getCurrentTime()).toFixed(2)} `); + player.seekTo(time, true); + } + } + + componentDidMount () { + //this.timer = setInterval(() => this.orchestrator(), ORCHESTRATOR_INTERVAL_MILLISECOND); + this.startOrchestrator(); + } + + componentWillUnmount () { + //clearInterval(this.timer); + this.stopOrchestrator(); + } + + startOrchestrator() { + //logger.debug("startOrchestrator"); + if (this.isOrchestrating) return; + this.isOrchestrating = true; + const loop = () => { + if (!this.isOrchestrating) return; + this.orchestrator(); + this.rafId = requestAnimationFrame(loop); + }; + this.rafId = requestAnimationFrame(loop); + } + + stopOrchestrator() { + //logger.debug("stopOrchestrator"); + this.isOrchestrating = false; + if (this.rafId) cancelAnimationFrame(this.rafId); + this.rafId = null; + } + + whichVideo = (videos, time) => { + const found = videos.find(video => video.timestamp <= time && video.clear >= time); + return found ? found : {url: "", timestamp: 0, clear: 0}; + } + + orchestrator () { + const { /*events, active, primaryPlaybackRate, primaryPlaybackVolume, primaryPlaybackMuted,*/ videos } = this.props; + const { playing, playbackRate } = this.state; + + if (!player.primary) return; + + this.time = player.primary.currentTime(); + + let primaryPlayerPlaying = true; + if (this.time === this.lastTime) { + primaryPlayerPlaying = false; + } + + //this.handleVolumeChange(primaryPlaybackVolume, primaryPlaybackMuted); // did not work...? + this.handleVolumeChange(player.primary.volume(), player.primary.muted()); + + this.lastTime = this.time; + this.primaryPlayerPlaying = primaryPlayerPlaying; + + if (/*active &&*/playing && !this.hasPlayedBefore && !this.autoPlayTimeout) { + this.autoPlayTimeout = setTimeout(this.autoPlayBlockDetected, AUTO_PLAY_BLOCK_DETECTION_TIMEOUT_SECONDS * 1000); + } + + const currentVideo = this.whichVideo(videos, this.time); + const index = getCurrentDataIndex(currentVideo.events, this.time); + if (index < 0 || currentVideo.clear < this.time) { + // when the primary player rewinds before the start of external video or finishes the video play. + this.lastEventPlaybackRate = 1; + } + + //if (active) { + if (currentVideo.url !== this.state.urlPlayed) { + this.setState({ urlPlayed: currentVideo.url }); + logger.debug(`external_video URLchange ${currentVideo.url} -> ${this.state.urlPlayed}`); + this.setState({ playing: false }); // When swapping to a different video. + } + // Check time consistency every ORCHESTRATOR_INTERVAL_MILLISECOND msec, and fix when drifted away too much + if (playing) { + let thisMovieTimeToBe; + if (index > -1 && currentVideo.events && currentVideo.events[index]) { + // thisMovieTimeToBe = MovieTimeToBe + (currentPlayerTime - eventTimeStamp) * playRate + // [movie time after calibration] [from the start of the movie] [how much sec from the timestamp of a update event] + thisMovieTimeToBe = parseFloat(currentVideo.events[index].time) + (this.time - currentVideo.events[index].timestamp) * currentVideo.events[index].rate; + logger.debug(`eventTimeStamp=${currentVideo.events[index].timestamp.toFixed(2)} MovieTimeToBe=${parseFloat(currentVideo.events[index].time).toFixed(2)} currentPlayerTime=${this.time.toFixed(2)} currentMovieTime=${this.player.getCurrentTime().toFixed(2)} rate=${currentVideo.events[index].rate}`); + logger.debug(`Calibrated movie time to be=${thisMovieTimeToBe.toFixed(2)} actual movie time=${this.player.getCurrentTime().toFixed(2)} inconsistency=${(thisMovieTimeToBe - this.player.getCurrentTime()).toFixed(2)} Type=${currentVideo.events[index].type}`); + } else if (currentVideo.events && currentVideo.events.length === 0) { + // Just a single video without pause, rate change, or whatever other events + thisMovieTimeToBe = this.time - currentVideo.timestamp; + } + this.seekTo(thisMovieTimeToBe); + } + //} + + logger.debug(`external_video: player url=${this.state.urlPlayed} time=${this.time.toFixed(2)} Playing=${playing} primaryPlayerPlaying=${primaryPlayerPlaying} PlaybackRate=${playbackRate}, events=${currentVideo.events}`); + + if (!primaryPlayerPlaying /*|| !active*/) { + this.handleOnPause(); + this.playerUpdateTime = -1; + return + } + + if (index > -1 && currentVideo.events && currentVideo.events[index] && currentVideo.events[index].type) + { + const {type, time, rate, playing} = currentVideo.events[index]; + + logger.debug(`External Video Event: type=${type} time=${time} rate=${rate} playing=${playing}`); + + const nextEvent = currentVideo.events[index+1]; + if (nextEvent && type === "stop" && nextEvent.type === "play" && (nextEvent.time - time) < IGNORE_STOP_CLOSE_TO_START_SECOND ){ + logger.debug(`external_video: player skipped "stop" event due to a close "play", interval=${nextEvent.time - time}`); + return; + } + + switch (type) { + case "stop": + this.handleOnPause(); + break; + case "play": + this.handleOnPlay(); + break; + case "playerUpdate": case "setPlaybackRate": // befor v3 + case "seek": case "playbackRateChange": // from v3 + if (this.playerUpdateTime !== time) { + this.seekTo(time); + playing ? this.handleOnPlay() : this.handleOnPause() + this.playerUpdateTime=time; + } + break; + default: + ; + } + // Play rate being adjusted every time. + this.lastEventPlaybackRate=rate; + } else if (index === -1 && (currentVideo.events && currentVideo.events.length === 0)) { + // start video without events, with the playing rate 1 (supposed to be) + this.lastEventPlaybackRate = 1; + this.handleOnPlay(); + } + // multiply the primary player's playing rate + this.setPlaybackRate(player.primary.playbackRate()); + } + + + render() { + + const { /*videoUrl, active,*/ intl/*, video*/ } = this.props; + const { playing, playbackRate, muted, autoPlayBlocked, volume, urlPlayed } = this.state; + + logger.debug(`Rendering ${urlPlayed}. Note this shouldn't be shown frequently!`); + return ( +
{ this.playerParent = ref; }} + > + {autoPlayBlocked + ? ( +

+ {intl.formatMessage(intlMessages.autoPlayWarning)} +

+ ) + : '' + } + + { this.player = ref; }} + width="100%" + height="100%" + /> + +
+ ); + + } +} + +export default (ExternalVideoPlayer); diff --git a/src/components/external-video-player/styles.css b/src/components/external-video-player/styles.css new file mode 100644 index 00000000..11cd3c00 --- /dev/null +++ b/src/components/external-video-player/styles.css @@ -0,0 +1,21 @@ +.autoPlayWarning { + position: absolute; + z-index: 100; + font-size: x-large; + color: white; + width: 100%; + background-color: rgba(6,23,42,0.5); + bottom: 20%; + vertical-align: middle; + text-align: center; + pointer-events: none; +} + +.externalVideos-wrapper { + display: flex; + position: absolute; + height: 100%; + position: absolute; + width: 95%; + left: 2.5%; +} diff --git a/src/components/player/content/index.js b/src/components/player/content/index.js index 6502f312..b22f3d20 100644 --- a/src/components/player/content/index.js +++ b/src/components/player/content/index.js @@ -6,6 +6,7 @@ import TldrawPresentationV2 from 'components/tldraw_v2'; import { getTldrawBbbVersion } from 'utils/tldraw'; import { useCurrentInterval, useShouldShowScreenShare } from 'components/utils/hooks'; import Screenshare from 'components/screenshare'; +import ExternalVideoPlayer from 'components/external-video-player'; import Thumbnails from 'components/thumbnails'; import FullscreenButton from 'components/player/buttons/fullscreen'; import { LAYOUT } from 'utils/constants'; @@ -13,6 +14,7 @@ import { isEqual } from 'utils/data/validators'; import layout from 'utils/layout'; import storage from 'utils/data/storage'; import './index.scss'; +import { useIntl } from 'react-intl'; import { gte as semverGte } from 'semver'; const Content = ({ @@ -34,6 +36,25 @@ const Content = ({ storage.panzooms.tldraw || storage.cursor.tldraw; + const RenderExternalVideo = () => { + const intl = useIntl(); + const { external_videos } = storage; + + return ( + + ); + } + let presentation; if (isTldrawWhiteboard) { @@ -73,6 +94,7 @@ const Content = ({ ): null} + {layout.external_videos ? RenderExternalVideo() : null}
+ +
+ ); + } + const logo = src.includes('logo'); return ( diff --git a/src/components/thumbnails/index.scss b/src/components/thumbnails/index.scss index f46413e5..0bf00149 100644 --- a/src/components/thumbnails/index.scss +++ b/src/components/thumbnails/index.scss @@ -41,5 +41,11 @@ $thumbnail-height: calc((#{$bottom-content-height} / 10) * 6); font-size: xxx-large; font-weight: var(--font-weight-semi-bold); } + + .external_video { + font-size: xxx-large; + font-weight: var(--font-weight-semi-bold); + } + } } diff --git a/src/components/utils/icon/index.scss b/src/components/utils/icon/index.scss index ff19f741..dee9222e 100644 --- a/src/components/utils/icon/index.scss +++ b/src/components/utils/icon/index.scss @@ -89,6 +89,10 @@ content: "\e930"; } +.icon-externalVideos:before { + content: "\e95d"; +} + .icon-dark:before { content: "\e9A0"; } diff --git a/src/config.js b/src/config.js index 4dcf2581..83f61b9b 100644 --- a/src/config.js +++ b/src/config.js @@ -27,6 +27,7 @@ const files = { shapes: 'shapes.svg', tldraw: 'tldraw.json', videos: 'external_videos.json', + externalVideos: 'external_videos.xml', layout: 'layout.xml', }; diff --git a/src/locales/messages/en.json b/src/locales/messages/en.json index 8e733f68..05f1a36a 100644 --- a/src/locales/messages/en.json +++ b/src/locales/messages/en.json @@ -39,5 +39,7 @@ "player.search.modal.title": "Search", "player.search.modal.subtitle": "Find presentation slides content", "player.thumbnails.wrapper.aria": "Thumbnails area", - "player.webcams.wrapper.aria": "Webcams area" + "player.webcams.wrapper.aria": "Webcams area", + "player.video.wrapper.aria": "Video area", + "player.externalVideo.autoPlayWarning": "We need your permission for playing audio" } diff --git a/src/locales/messages/ja.json b/src/locales/messages/ja.json index 205ae2b8..ca9ae413 100644 --- a/src/locales/messages/ja.json +++ b/src/locales/messages/ja.json @@ -39,5 +39,7 @@ "player.search.modal.title": "検索", "player.search.modal.subtitle": "スライド上のテキストを探す", "player.thumbnails.wrapper.aria": "サムネイルエリア", - "player.webcams.wrapper.aria": "ウェブカムエリア" + "player.webcams.wrapper.aria": "ウェブカムエリア", + "player.video.wrapper.aria": "ビデオエリア", + "player.externalVideo.autoPlayWarning": "音声の再生には許可が必要です" } diff --git a/src/styles/assets/icons.woff b/src/styles/assets/icons.woff index 4cbb70a1..5ace36bc 100644 Binary files a/src/styles/assets/icons.woff and b/src/styles/assets/icons.woff differ diff --git a/src/utils/builder.js b/src/utils/builder.js index 0a39f37e..6bc9c7d0 100644 --- a/src/utils/builder.js +++ b/src/utils/builder.js @@ -221,6 +221,12 @@ const buildThumbnails = slides => { src: ID.SCREENSHARE, timestamp, }); + } else if (src.includes(ID.EXTERNAL_VIDEOS)) { + result.push({ + id, + src: ID.EXTERNAL_VIDEOS, + timestamp, + }); } else { result.push({ id, @@ -514,6 +520,35 @@ const buildScreenshare = result => { return data; }; +const buildExternalVideos = result => { + let data = []; + const { recording } = result; + + if (hasProperty(recording, 'video')) { + const v = recording.video; + const videos = Array.isArray(v) ? v : [v]; + data = videos.map(video => { + const events = Array.isArray(video.event) + ? video.event.map(event => ({ + timestamp: parseFloat(event._timestamp), + type: event._type, + time: event._time, + rate: parseFloat(event._rate), + playing: (event._playing === 'true'), + })) + : []; + return { + timestamp: parseFloat(video._start_timestamp), + clear: parseFloat(video._stop_timestamp), + url: video._url, + events, + }; + }); + } + + return data; +}; + const build = (filename, value) => { return new Promise((resolve, reject) => { let data; @@ -580,6 +615,9 @@ const build = (filename, value) => { case config.screenshare: data = buildScreenshare(result); break; + case config.externalVideos: + data = buildExternalVideos(result); + break; case config.shapes: data = buildShapes(result); break; diff --git a/src/utils/constants.js b/src/utils/constants.js index 6f9606c4..93713954 100644 --- a/src/utils/constants.js +++ b/src/utils/constants.js @@ -41,6 +41,7 @@ const ID = { POLLS: 'polls', PRESENTATION: 'presentation', SCREENSHARE: 'screenshare', + EXTERNAL_VIDEOS: 'externalVideos', SEARCH: 'search', SETTINGS: 'settings', SHAPES: 'shapes', @@ -62,6 +63,7 @@ const CONTENT = [ ID.VIDEOS, ID.NOTES, ID.SCREENSHARE, + ID.EXTERNAL_VIDEOS, ID.CAPTIONS, ]; diff --git a/src/utils/data/index.js b/src/utils/data/index.js index 9edab665..883d838e 100644 --- a/src/utils/data/index.js +++ b/src/utils/data/index.js @@ -86,10 +86,17 @@ const getPads = (n) => { const getCurrentContent = (time) => { const { SCREENSHARE, + EXTERNAL_VIDEOS, PRESENTATION, } = ID; + + let content=PRESENTATION; - const content = isEnabled(storage.screenshare, time) ? SCREENSHARE : PRESENTATION; + if (isEnabled(storage.screenshare, time)) { + content=SCREENSHARE; + } else if (isEnabled(storage.external_videos, time)) { + content=EXTERNAL_VIDEOS; + } return content; }; diff --git a/src/utils/data/storage.js b/src/utils/data/storage.js index d8022d5a..379b0a5f 100644 --- a/src/utils/data/storage.js +++ b/src/utils/data/storage.js @@ -165,6 +165,7 @@ const storage = { videos: hasProperty(DATA, ID.VIDEOS), presentation: hasProperty(DATA, ID.SHAPES), screenshare: hasProperty(DATA, ID.SCREENSHARE), + externalVideos: hasProperty(DATA, ID.EXTERNAL_VIDEOS), layoutSwap: hasProperty(DATA, ID.LAYOUT), }; }, @@ -177,6 +178,7 @@ const storage = { videos: !isEmpty(this.videos), presentation: hasPresentation(this.slides), screenshare: !isEmpty(this.screenshare), + externalVideos: !isEmpty(this.external_videos), layoutSwap: !isEmpty(this.layoutSwap), }; }, @@ -224,6 +226,9 @@ const storage = { get screenshare() { return DATA[ID.SCREENSHARE]; }, + get external_videos() { + return DATA[ID.EXTERNAL_VIDEOS]; + }, get shapes() { return DATA[ID.SHAPES]; }, diff --git a/src/utils/layout.js b/src/utils/layout.js index 29f75803..3be00503 100644 --- a/src/utils/layout.js +++ b/src/utils/layout.js @@ -71,8 +71,11 @@ const layout = { get screenshare() { return this.content.screenshare; }, + get external_videos() { + return this.content.externalVideos; + }, get single() { - return !this.content.presentation && !this.content.screenshare; + return !this.content.presentation && !this.content.screenshare && !this.content.externalVideos; }, hasFullscreenButton: function (content, swap) { if (!this.control || !controls.fullscreen) return false; diff --git a/src/utils/player.js b/src/utils/player.js index 96f9372b..59372452 100644 --- a/src/utils/player.js +++ b/src/utils/player.js @@ -11,6 +11,10 @@ const player = { get screenshare() { return PLAYERS[ID.SCREENSHARE]; }, + get external_videos() { + //Not really used.. + return PLAYERS[ID.EXTERNAL_VIDEOS]; + }, get synchronizer() { return SYNCHRONIZER; }, diff --git a/src/utils/synchronizer.js b/src/utils/synchronizer.js index 57ff18f8..04c185fe 100644 --- a/src/utils/synchronizer.js +++ b/src/utils/synchronizer.js @@ -36,10 +36,14 @@ const EVENTS = [ ]; export default class Synchronizer { - constructor(primary, secondary) { + constructor(primary, secondary/*, externalVideos = null*/) { this.primary = primary; this.secondary = secondary; + //if (externalVideos) { + // this.externalVideos = externalVideos; + //} + this.status = { primary: 'waiting', secondary: 'waiting',