From 85a14b51015a7dd4a792f99b785eb3fdf5c96f4a Mon Sep 17 00:00:00 2001 From: blue Date: Mon, 17 Dec 2018 20:15:58 +0300 Subject: [PATCH] Barely working basic playback --- corax/models/player.cpp | 7 +- lib/wModel/CMakeLists.txt | 3 + lib/wModel/button.cpp | 2 +- lib/wModel/file/audio.cpp | 80 ++++++++++++++++ lib/wModel/file/audio.h | 36 +++++++ lib/wModel/file/file.cpp | 26 ++++- lib/wModel/file/file.h | 1 + lib/wModel/model.h | 1 + lib/wType/blob.cpp | 14 +++ lib/wType/blob.h | 2 + libjs/utils/CMakeLists.txt | 1 + libjs/utils/stateMachine.js | 33 +++++++ libjs/wController/CMakeLists.txt | 1 + libjs/wController/file/audio.js | 69 ++++++++++++++ libjs/wController/player.js | 132 ++++++++++++++++++++++++++ lorgar/lib/utils/CMakeLists.txt | 1 + lorgar/lib/wController/CMakeLists.txt | 1 + lorgar/views/button.js | 2 - lorgar/views/view.js | 18 +--- 19 files changed, 406 insertions(+), 24 deletions(-) create mode 100644 lib/wModel/file/audio.cpp create mode 100644 lib/wModel/file/audio.h create mode 100644 libjs/utils/stateMachine.js create mode 100644 libjs/wController/file/audio.js diff --git a/corax/models/player.cpp b/corax/models/player.cpp index 1f480e5..4c58204 100644 --- a/corax/models/player.cpp +++ b/corax/models/player.cpp @@ -4,7 +4,7 @@ M::Player::Player(const W::Address& address, QObject* parent): M::Model(address, parent), controls(), views(), - playPauseBtn(new M::Button(address + W::Address{u"play"})), + playPauseBtn(new M::Button(address + W::Address{u"Play"})), _queueView(new M::List(address + W::Address{u"queueView"})), _queue(), current(0), @@ -93,7 +93,7 @@ void M::Player::h_get(const W::Event& ev) void M::Player::onPlayPauseBtn() { if (playing) { - playPauseBtn->setLabel(W::String(u"Pause")); + playPauseBtn->setLabel(W::String(u"Play")); playing = false; switch (mode) { @@ -102,7 +102,7 @@ void M::Player::onPlayPauseBtn() break; } } else { - playPauseBtn->setLabel(W::String(u"Play")); + playPauseBtn->setLabel(W::String(u"Pause")); playing = true; switch (mode) { @@ -135,6 +135,7 @@ void M::Player::h_queue(const W::Event& ev) res->insert(u"remove", new W::Vector()); broadcast(res, W::Address{u"viewsChange"}); + playPauseBtn->setEnabled(true); } else { _queue.push_back(song); _queueView->push(song->getAddress()); diff --git a/lib/wModel/CMakeLists.txt b/lib/wModel/CMakeLists.txt index fb70a7b..cc83c8f 100644 --- a/lib/wModel/CMakeLists.txt +++ b/lib/wModel/CMakeLists.txt @@ -16,6 +16,7 @@ set(HEADERS catalogue.h button.h file/file.h + file/audio.h ) set(SOURCES @@ -28,6 +29,7 @@ set(SOURCES catalogue.cpp button.cpp file/file.cpp + file/audio.cpp ) add_library(wModel STATIC ${HEADERS} ${SOURCES}) @@ -36,3 +38,4 @@ target_link_libraries(wModel Qt5::Core) target_link_libraries(wModel wServerUtils) target_link_libraries(wModel wType) target_link_libraries(wModel wController) +target_link_libraries(wModel mad) diff --git a/lib/wModel/button.cpp b/lib/wModel/button.cpp index 649f999..9f22ea4 100644 --- a/lib/wModel/button.cpp +++ b/lib/wModel/button.cpp @@ -52,7 +52,7 @@ void M::Button::setEnabled(bool p_enabled) enabled = p_enabled; if (registered) { W::Vocabulary* vc = new W::Vocabulary(); - vc->insert(u"enable", new W::Boolean(enabled)); + vc->insert(u"enabled", new W::Boolean(enabled)); broadcast(vc, W::Address{u"setEnabled"}); } } diff --git a/lib/wModel/file/audio.cpp b/lib/wModel/file/audio.cpp new file mode 100644 index 0000000..86fb0e1 --- /dev/null +++ b/lib/wModel/file/audio.cpp @@ -0,0 +1,80 @@ +#include "audio.h" + +#include + +M::Audio::Audio(W::Blob* p_file, const W::Address& addr, QObject* parent): + File(p_file, addr, parent), + frames() +{ + W::Handler* requestFrames = W::Handler::create(address + W::Address({u"requestFrames"}), this, &M::Audio::_h_requestFrames); + + addHandler(requestFrames); +} + +M::Audio::~Audio() +{} + +M::Model::ModelType M::Audio::getType() const +{ + return type; +} + +void M::Audio::initAdditional(const W::String& p_mime) +{ + File::initAdditional(p_mime); //TODO handle other than mp3 formats + + mad_stream stream; + mad_header header; + mad_stream_init(&stream); + mad_header_init(&header); + + mad_stream_buffer(&stream, file->uchar(), file->size()); + + uint64_t length = 0; + uint64_t tBits = 0; + while(stream.error != MAD_ERROR_BUFLEN) { //TODO handle other errors; + + int success = mad_header_decode(&header, &stream); + if (success == 0) { + frames.emplace_back(stream.this_frame - stream.buffer, stream.next_frame - stream.this_frame); + + length += header.duration.seconds * MAD_TIMER_RESOLUTION + header.duration.fraction; + tBits += header.bitrate; + } + } + + additional.insert(u"duration", new W::Uint64(length / MAD_TIMER_RESOLUTION)); + additional.insert(u"bitrate", new W::Uint64(tBits / frames.size())); + additional.insert(u"framesAmount", new W::Uint64(frames.size())); +} + +void M::Audio::h_requestFrames(const W::Event& ev) +{ + const W::Vocabulary& vc = static_cast(ev.getData()); + const W::Uint64& index = static_cast(vc.at(u"index")); + const W::Uint64& amount = static_cast(vc.at(u"amount")); + + W::Vocabulary* evc = new W::Vocabulary(); + if (index + amount > frames.size()) { + evc->insert(u"result", new W::Uint64(1)); + } else { + evc->insert(u"result", new W::Uint64(0)); + uint64_t start = 0; + uint64_t size = 0; + bool first = true; + for (int i = 0; i < amount; ++i) { + const std::pair& pair = frames[index + i]; + if (first) { + start = pair.first; + first = false; + } + size += pair.second; + } + + evc->insert(u"data", file->slice(start, size)); + evc->insert(u"amount", new W::Uint64(amount)); + + } + + response(evc, W::Address{u"responseFrames"}, ev); +} diff --git a/lib/wModel/file/audio.h b/lib/wModel/file/audio.h new file mode 100644 index 0000000..070e31c --- /dev/null +++ b/lib/wModel/file/audio.h @@ -0,0 +1,36 @@ +#ifndef M_AUDIO_H +#define M_AUDIO_H + +#include + +#include +#include + +namespace M { + + /** + * @todo write docs + */ + class Audio : public File { + friend class File; + protected: + Audio(W::Blob* p_file, const W::Address& addr, QObject* parent = 0); + + public: + ~Audio(); + + M::Model::ModelType getType() const override; + static const M::Model::ModelType type = audio; + + protected: + void initAdditional(const W::String& p_mime) override; + + handler(requestFrames); + + private: + std::deque> frames; + }; + +} + +#endif // M_AUDIO_H diff --git a/lib/wModel/file/file.cpp b/lib/wModel/file/file.cpp index 083ec62..5efa9bd 100644 --- a/lib/wModel/file/file.cpp +++ b/lib/wModel/file/file.cpp @@ -1,7 +1,13 @@ #include "file.h" #include +#include "audio.h" + QMimeDatabase M::File::mimeDB; +const std::map M::File::mimeMap = { + {"image/jpeg", M::Model::file}, + {"audio/mpeg", M::Model::audio} +}; M::File::File(W::Blob* p_file, const W::Address& addr, QObject* parent): M::Model(addr, parent), @@ -79,8 +85,24 @@ M::File * M::File::create(W::Blob* blob, const W::Address& addr, QObject* parent M::File* out; QMimeType mt = mimeDB.mimeTypeForData(blob->byteArray()); - out = new File(blob, addr, parent); - out->initAdditional(W::String(mt.name().toStdString())); + const QString& mime = mt.name(); + std::map::const_iterator itr = mimeMap.find(mime); + + M::Model::ModelType modelType = M::Model::file; + if (itr != mimeMap.end()) { + modelType = itr->second; + } + + switch (modelType) { + case Model::audio: + out = new Audio(blob, addr, parent); + break; + default: + out = new File(blob, addr, parent); + break; + } + + out->initAdditional(W::String(mime.toStdString())); return out; } diff --git a/lib/wModel/file/file.h b/lib/wModel/file/file.h index 7a3d79d..8c50440 100644 --- a/lib/wModel/file/file.h +++ b/lib/wModel/file/file.h @@ -38,6 +38,7 @@ namespace M { W::Blob* file; static QMimeDatabase mimeDB; + static const std::map mimeMap; }; } diff --git a/lib/wModel/model.h b/lib/wModel/model.h index c10b173..e7f9742 100644 --- a/lib/wModel/model.h +++ b/lib/wModel/model.h @@ -36,6 +36,7 @@ namespace M { attributes = 50, file, resourceCache, + audio, player = 107 }; diff --git a/lib/wType/blob.cpp b/lib/wType/blob.cpp index 02991c2..9c08382 100644 --- a/lib/wType/blob.cpp +++ b/lib/wType/blob.cpp @@ -144,3 +144,17 @@ const QByteArray & W::Blob::byteArray() const return qDataView; } +const unsigned char * W::Blob::uchar() const +{ + return (unsigned char*) data; +} + +W::Blob* W::Blob::slice(uint64_t start, uint64_t length) const +{ + char* n_data = new char[length]; + for (int i = 0; i < length; ++i) { + n_data[i] = data[start + i]; + } + + return new W::Blob(length, n_data); +} diff --git a/lib/wType/blob.h b/lib/wType/blob.h index e8fdb55..8995974 100644 --- a/lib/wType/blob.h +++ b/lib/wType/blob.h @@ -22,6 +22,7 @@ namespace W size_type size() const override; objectType getType() const override; + Blob* slice(uint64_t start, uint64_t length = 0) const; bool operator==(const W::Object & other) const override; @@ -33,6 +34,7 @@ namespace W static const objectType type = blob; const QByteArray& byteArray() const; + const unsigned char* uchar() const; protected: bool hasData; diff --git a/libjs/utils/CMakeLists.txt b/libjs/utils/CMakeLists.txt index a0018e8..7ec6696 100644 --- a/libjs/utils/CMakeLists.txt +++ b/libjs/utils/CMakeLists.txt @@ -4,3 +4,4 @@ configure_file(class.js class.js) configure_file(subscribable.js subscribable.js) configure_file(globalMethods.js globalMethods.js) configure_file(enum.js enum.js) +configure_file(stateMachine.js stateMachine) diff --git a/libjs/utils/stateMachine.js b/libjs/utils/stateMachine.js new file mode 100644 index 0000000..b15f0cd --- /dev/null +++ b/libjs/utils/stateMachine.js @@ -0,0 +1,33 @@ +"use strict"; + +var Subscribable = require("./subscribable"); + +var StateMachine = Subscribable.inherit({ + className: "StateMachine", + constructor: function (initialState, graph) { + Subscribable.fn.constructor.call(this); + + this._state = initialState; + this._graph = graph; + }, + manipulation: function (name) { + var newState = this._graph[this._state][name]; + if (newState) { + var oldState = this._state; + this._state = newState; + + this.trigger("stateChanged", { + newState: newState, + manipulation: name, + oldState: oldState + }); + } else { + this.trigger("stateMissed"); + } + }, + state: function () { + return this._state; + } +}); + +module.exports = StateMachine; diff --git a/libjs/wController/CMakeLists.txt b/libjs/wController/CMakeLists.txt index 8ecb88b..f51dfbb 100644 --- a/libjs/wController/CMakeLists.txt +++ b/libjs/wController/CMakeLists.txt @@ -17,5 +17,6 @@ configure_file(localModel.js localModel.js) configure_file(catalogue.js catalogue.js) configure_file(imagePane.js imagePane.js) configure_file(file/file.js file/file.js) +configure_file(file/audio.js file/audio.js) configure_file(player.js player.js) configure_file(imageById.js imageById.js) diff --git a/libjs/wController/file/audio.js b/libjs/wController/file/audio.js new file mode 100644 index 0000000..1b77cf3 --- /dev/null +++ b/libjs/wController/file/audio.js @@ -0,0 +1,69 @@ +"use strict"; + +var File = require("./file"); +var Vocabulary = require("../../wType/vocabulary"); +var Uint64 = require("../../wType/uint64"); + +var Audio = File.inherit({ + className: "Audio", + constructor: function Audio(addr) { + File.fn.constructor.call(this, addr); + + this._loadedFrames = 0; + this._totalFrames = 0; + this._waitingForFrames = false; + this._portions = []; + + this.addHandler("responseFrames"); + }, + hasMore: function() { + return this._totalFrames > this._loadedFrames; + }, + _getAdditional: function(add) { + var ac = File.fn._getAdditional.call(this, add); + + if (ac) { + this._loadedFrames = 0; + this._totalFrames = this._additional.at("framesAmount").valueOf(); + this._waitingForFrames = false; + } + + this.initialized = true; + return ac; + }, + _h_responseFrames: function(ev) { + if (this._waitingForFrames === true) { + var data = ev.getData(); + + var success = data.at("result").valueOf(); + if (success === 0) { + var amount = data.at("amount").valueOf(); + var blob = data.at("data").clone(); + this._loadedFrames += amount; + this._waitingForFrames = false; + this._portions.push(blob); + this.trigger("newFrames", blob); + } + } + }, + requestMore: function() { + if (!this._waitingForFrames) { + if (this._registered && this._subscribed) { + var allowed = this._totalFrames - this._loadedFrames; + + if (allowed > 0) { + var vc = new Vocabulary(); + vc.insert("index", new Uint64(this._loadedFrames)); + vc.insert("amount", new Uint64(Math.min(framePortion, allowed))); + + this.send(vc, "requestFrames"); + this._waitingForFrames = true; + } + } + } + } +}); + +var framePortion = 10; + +module.exports = Audio; diff --git a/libjs/wController/player.js b/libjs/wController/player.js index d7bea05..32021bf 100644 --- a/libjs/wController/player.js +++ b/libjs/wController/player.js @@ -7,7 +7,10 @@ var Controller = require("./controller"); var Button = require("./button"); var ImageById = require("./imageById"); var Vocabulary = require("./vocabulary"); +var Audio = require("./file/audio"); + var Enum = require("../utils/enum"); +var StateMachine = require("../utils/stateMachine"); var Player = Controller.inherit({ className: "Player", @@ -17,9 +20,24 @@ var Player = Controller.inherit({ this.controls = Object.create(null); this.views = Object.create(null); this.mode = PlayerMode.straight.playback; + this._audio = null; + this._sound = new window.Audio(); + this._ctx = new AudioContext(); + this._currentTime = 0; + this._createStateMachine(); + this._proxySchedule = this._schedule.bind(this); this.addHandler("get"); this.addHandler("viewsChange"); + this.addHandler("play"); + this.addHandler("pause"); + }, + destructor: function() { + this._fsm.destructor(); + this._sound.pause(); + this._ctx.close(); + + Controller.fn.destructor.call(this); }, _addControl: function(type, address) { var t = type.valueOf(); @@ -87,6 +105,10 @@ var Player = Controller.inherit({ this.trigger("newElement", ctrl, t); } }, + _createStateMachine: function() { + this._fsm = new StateMachine("initial", graphs[this.mode]); + this._fsm.on("stateChanged", this._onStateChanged, this); + }, _h_get: function(ev) { var data = ev.getData(); @@ -117,6 +139,12 @@ var Player = Controller.inherit({ this.initialized = true; this.trigger("data"); }, + _h_pause: function(ev) { + this._fsm.manipulation("plause"); + }, + _h_play: function(ev) { + this._fsm.manipulation("play"); + }, _h_viewsChange: function(ev) { var data = ev.getData(); @@ -135,6 +163,15 @@ var Player = Controller.inherit({ this._addView(vc.at("type"), vc.at("address")); } }, + _onAudioNewFrames: function(frames) { + this._ctx.decodeAudioData(frames.valueOf(), this._proxySchedule); + this._fsm.manipulation("newFrames"); + if (this._audio.hasMore()) { + this._audio.requestMore(); + } else { + this._fsm.manipulation("noMoreFrames"); + } + }, _onNewPlayBackElement: function(key, element) { switch (key) { case "image": @@ -142,6 +179,14 @@ var Player = Controller.inherit({ this._addView(new Uint64(ItemType.straight.picture), address); address.destructor(); break; + case "audio": + if (this.mode === PlayerMode.straight.playback) { + this._audio = new Audio(new Address(["music", element.toString()])); + this.addForeignController("Corax", this._audio); + this._audio.on("newFrames", this._onAudioNewFrames, this); + this._fsm.manipulation("controller"); + } + break; } }, _onNewRemoveBackElement: function(key) { @@ -149,6 +194,51 @@ var Player = Controller.inherit({ case "image": this._removeView(new Uint64(ItemType.straight.picture)); break; + case "audio": + this.removeForeignController(this._audio); + this._audio.destructor(); + this._audio = null; + } + }, + _onStateChanged: function(e) { + switch (e.newState) { + case "initial": + break; + case "initialPlaying": + break; + case "hasController": + break; + case "hasControllerPlaying": + if (this._audio.hasMore()) { + this._audio.requestMore(); + } else { + this._fsm.manipulation("noMoreFrames"); + } + break; + case "paused": + switch (e.oldState) { + case "playing": + this._sound.pause(); + break; + } + break; + case "pausedAllLoaded": + switch (e.oldState) { + case "playingAllLoaded": + this._sound.pause(); + break; + } + break; + case "playing": + this._sound.play(); + break; + case "playingAllLoaded": + switch (e.oldState) { + case "pausedAllLoaded": + this._sound.play(); + break; + } + break; } }, _removeControl: function(type) { @@ -156,6 +246,14 @@ var Player = Controller.inherit({ }, _removeView: function(type) { //TODO + }, + _schedule: function(buffer) { + var source = this._ctx.createBufferSource(); + source.buffer = buffer; + source.connect(this._ctx.destination); + + source.start(this._currentTime); + this._currentTime += buffer.duration; } }); @@ -170,4 +268,38 @@ PlayerMode.add("playback"); Player.ItemType = ItemType; +var graphs = Object.create(null); +graphs[PlayerMode.straight.playback] = { + "initial": { + controller: "hasController", + play: "initialPlaying" + }, + "initialPlaying": { + pause: "initial", + controller: "hasControllerPlaying" + }, + "hasController": { + newFrames: "paused", + play: "hasControllerPlaying" + }, + "hasControllerPlaying": { + newFrames: "playing", + pause: "hasController" + }, + "paused": { + play: "playing", + noMoreFrames: "pausedAllLoaded" + }, + "pausedAllLoaded": { + play: "playingAllLoaded" + }, + "playing": { + pause: "pause", + noMoreFrames: "playingAllLoaded" + }, + "playingAllLoaded": { + pause: "pausedAllLoaded" + } +} + module.exports = Player; diff --git a/lorgar/lib/utils/CMakeLists.txt b/lorgar/lib/utils/CMakeLists.txt index fc6a3f2..c988215 100644 --- a/lorgar/lib/utils/CMakeLists.txt +++ b/lorgar/lib/utils/CMakeLists.txt @@ -4,3 +4,4 @@ add_jslib(utils/class.js lib/utils/class ${LORGAR_DIR} browser) add_jslib(utils/subscribable.js lib/utils/subscribable ${LORGAR_DIR} browser) add_jslib(utils/globalMethods.js lib/utils/globalMethods ${LORGAR_DIR} browser) add_jslib(utils/enum.js lib/utils/enum ${LORGAR_DIR} browser) +add_jslib(utils/stateMachine.js lib/utils/stateMachine ${LORGAR_DIR} browser) diff --git a/lorgar/lib/wController/CMakeLists.txt b/lorgar/lib/wController/CMakeLists.txt index ad3451b..7ed96b3 100644 --- a/lorgar/lib/wController/CMakeLists.txt +++ b/lorgar/lib/wController/CMakeLists.txt @@ -16,6 +16,7 @@ add_jslib(wController/attributes.js lib/wController/attributes ${LORGAR_DIR} bro add_jslib(wController/localModel.js lib/wController/localModel ${LORGAR_DIR} browser) add_jslib(wController/imagePane.js lib/wController/imagePane ${LORGAR_DIR} browser) add_jslib(wController/file/file.js lib/wController/file/file ${LORGAR_DIR} browser) +add_jslib(wController/file/audio.js lib/wController/file/audio ${LORGAR_DIR} browser) add_jslib(wController/image.js lib/wController/image ${LORGAR_DIR} browser) add_jslib(wController/button.js lib/wController/button ${LORGAR_DIR} browser) add_jslib(wController/player.js lib/wController/player ${LORGAR_DIR} browser) diff --git a/lorgar/views/button.js b/lorgar/views/button.js index d417da2..05bb72a 100644 --- a/lorgar/views/button.js +++ b/lorgar/views/button.js @@ -58,10 +58,8 @@ if (this._enabled !== enabled) { this._enabled = enabled; if (this._enabled) { - this.addClass("hoverable"); this.removeClass("disabled"); } else { - this.removeClass("hoverable"); this.addClass("disabled"); } } diff --git a/lorgar/views/view.js b/lorgar/views/view.js index 70a624c..8016b1f 100644 --- a/lorgar/views/view.js +++ b/lorgar/views/view.js @@ -78,11 +78,7 @@ Subscribable.fn.destructor.call(this); }, "addClass": function(className) { - var arr = this._e.className.split(" "); - if (arr.indexOf(className) === -1) { - arr.push(className); - this._e.className = arr.join(" "); - } + this._e.classList.add(className); }, "_applyProperties": function() { for (var i = 0; i < this._f.properties.length; ++i) { @@ -141,17 +137,7 @@ } }, "removeClass": function(className) { - var arr = this._e.className.split(" "); - var index = arr.indexOf(className) - var toJoin = false; - while (index !== -1) { - arr.splice(index, 1); - index = arr.indexOf(className) - toJoin = true; - } - if (toJoin) { - this._e.className = arr.join(" "); - } + this._e.classList.remove(className); }, "_resetTheme": function() { this._onClearProperties();