667
edits
Changes
Created page with '*** needs to get applied to mozilla-central rev 683dfdc4adf0 diff --git a/content/base/src/nsContentUtils.cpp b/content/base/src/nsContentUtils.cpp --- a/content/base/src/nsCont...'
*** needs to get applied to mozilla-central rev 683dfdc4adf0
diff --git a/content/base/src/nsContentUtils.cpp b/content/base/src/nsContentUtils.cpp
--- a/content/base/src/nsContentUtils.cpp
+++ b/content/base/src/nsContentUtils.cpp
@@ -486,16 +486,17 @@ nsContentUtils::InitializeEventTable() {
{ &nsGkAtoms::oncanplaythrough, { NS_CANPLAYTHROUGH, EventNameType_HTML }},
{ &nsGkAtoms::onseeking, { NS_SEEKING, EventNameType_HTML }},
{ &nsGkAtoms::onseeked, { NS_SEEKED, EventNameType_HTML }},
{ &nsGkAtoms::ontimeupdate, { NS_TIMEUPDATE, EventNameType_HTML }},
{ &nsGkAtoms::onended, { NS_ENDED, EventNameType_HTML }},
{ &nsGkAtoms::onratechange, { NS_RATECHANGE, EventNameType_HTML }},
{ &nsGkAtoms::ondurationchange, { NS_DURATIONCHANGE, EventNameType_HTML }},
{ &nsGkAtoms::onvolumechange, { NS_VOLUMECHANGE, EventNameType_HTML }},
+ { &nsGkAtoms::onaudiowritten, { NS_AUDIOWRITTEN, EventNameType_HTML }},
#endif //MOZ_MEDIA
{ &nsGkAtoms::onMozAfterPaint, { NS_AFTERPAINT, EventNameType_None }},
{ &nsGkAtoms::onMozScrolledAreaChanged, { NS_SCROLLEDAREACHANGED, EventNameType_None }},
// Simple gesture events
{ &nsGkAtoms::onMozSwipeGesture, { NS_SIMPLE_GESTURE_SWIPE, EventNameType_None } },
{ &nsGkAtoms::onMozMagnifyGestureStart, { NS_SIMPLE_GESTURE_MAGNIFY_START, EventNameType_None } },
{ &nsGkAtoms::onMozMagnifyGestureUpdate, { NS_SIMPLE_GESTURE_MAGNIFY_UPDATE, EventNameType_None } },
diff --git a/content/base/src/nsGkAtomList.h b/content/base/src/nsGkAtomList.h
--- a/content/base/src/nsGkAtomList.h
+++ b/content/base/src/nsGkAtomList.h
@@ -1639,16 +1639,18 @@ GK_ATOM(seeking, "seeking")
GK_ATOM(seeked, "seeked")
GK_ATOM(timeupdate, "timeupdate")
GK_ATOM(ended, "ended")
GK_ATOM(canplay, "canplay")
GK_ATOM(canplaythrough, "canplaythrough")
GK_ATOM(ratechange, "ratechange")
GK_ATOM(durationchange, "durationchange")
GK_ATOM(volumechange, "volumechange")
+GK_ATOM(onaudiowritten, "onaudiowritten")
+GK_ATOM(audiowritten, "audiowritten")
#endif
// Frame property names
GK_ATOM(boxMetricsProperty, "BoxMetricsProperty") // nsBoxLayoutMetrics*
GK_ATOM(changeListProperty, "ChangeListProperty") // void*
GK_ATOM(collapseOffsetProperty, "CollapseOffsetProperty") // nsPoint*
GK_ATOM(computedOffsetProperty, "ComputedOffsetProperty") // nsPoint*
GK_ATOM(floatContinuationProperty, "FloatContinuationProperty") // nsFrameList*
diff --git a/content/events/public/nsIPrivateDOMEvent.h b/content/events/public/nsIPrivateDOMEvent.h
--- a/content/events/public/nsIPrivateDOMEvent.h
+++ b/content/events/public/nsIPrivateDOMEvent.h
@@ -45,16 +45,17 @@
{ 0xb0, 0x49, 0xdf, 0x9f, 0xc3, 0x6b, 0x56, 0xd4 } }
class nsIDOMEventTarget;
class nsIDOMEvent;
class nsEvent;
class nsCommandEvent;
class nsPresContext;
class nsInvalidateRequestList;
+class nsIDOMAudioData;
class nsIPrivateDOMEvent : public nsISupports
{
public:
NS_DECLARE_STATIC_IID_ACCESSOR(NS_IPRIVATEDOMEVENT_IID)
NS_IMETHOD DuplicatePrivateData() = 0;
NS_IMETHOD SetTarget(nsIDOMEventTarget* aTarget) = 0;
@@ -106,13 +107,21 @@ NS_NewDOMMessageEvent(nsIDOMEvent** aIns
nsresult
NS_NewDOMProgressEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsEvent* aEvent);
// This empties aInvalidateRequests.
nsresult
NS_NewDOMNotifyPaintEvent(nsIDOMEvent** aResult, nsPresContext* aPresContext,
nsEvent* aEvent,
PRUint32 aEventType = 0,
nsInvalidateRequestList* aInvalidateRequests = nsnull);
+
+nsresult
+NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aResult, nsPresContext* aPresContext,
+ nsEvent* aEvent,
+ PRUint32 aEventType = 0,
+ nsIDOMAudioData* aFrameBuffer = nsnull,
+ nsIDOMAudioData* aSpectrum = nsnull);
+
nsresult
NS_NewDOMSimpleGestureEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsSimpleGestureEvent* aEvent);
nsresult
NS_NewDOMScrollAreaEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsScrollAreaEvent* aEvent);
#endif // nsIPrivateDOMEvent_h__
diff --git a/content/events/src/Makefile.in b/content/events/src/Makefile.in
--- a/content/events/src/Makefile.in
+++ b/content/events/src/Makefile.in
@@ -73,16 +73,18 @@ CPPSRCS = \
nsPLDOMEvent.cpp \
nsEventDispatcher.cpp \
nsIMEStateManager.cpp \
nsContentEventHandler.cpp \
nsEventListenerService.cpp \
nsDOMProgressEvent.cpp \
nsDOMDataTransfer.cpp \
nsDOMNotifyPaintEvent.cpp \
+ nsDOMNotifyAudioWrittenEvent.cpp \
+ nsDOMAudioData.cpp \
nsDOMSimpleGestureEvent.cpp \
nsDOMEventTargetHelper.cpp \
nsDOMScrollAreaEvent.cpp \
$(NULL)
# we don't want the shared lib, but we want to force the creation of a static lib.
FORCE_STATIC_LIB = 1
diff --git a/content/events/src/nsDOMAudioData.cpp b/content/events/src/nsDOMAudioData.cpp
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMAudioData.cpp
@@ -0,0 +1,34 @@
+#include "nsDOMAudioData.h"
+#include "nsCOMPtr.h"
+#include "nsContentUtils.h"
+
+NS_INTERFACE_TABLE_HEAD(nsDOMAudioData)
+ NS_INTERFACE_TABLE1(nsDOMAudioData, nsIDOMAudioData)
+ NS_INTERFACE_TABLE_TO_MAP_SEGUE
+ NS_INTERFACE_MAP_ENTRY_CONTENT_CLASSINFO(AudioData)
+NS_INTERFACE_MAP_END
+
+NS_IMPL_ADDREF(nsDOMAudioData)
+NS_IMPL_RELEASE(nsDOMAudioData)
+
+
+nsDOMAudioData::nsDOMAudioData(nsTArray<float>& aData)
+{
+ mData.SwapElements(aData);
+}
+
+nsDOMAudioData::~nsDOMAudioData()
+{
+}
+
+NS_IMETHODIMP nsDOMAudioData::GetLength(PRUint32 *aLength)
+{
+ *aLength = mData.Length();
+ return NS_OK;
+}
+
+NS_IMETHODIMP nsDOMAudioData::Item(PRUint32 aIndex, float *retval NS_OUTPARAM)
+{
+ *retval = GetItemAt(aIndex);
+ return NS_OK;
+}
diff --git a/content/events/src/nsDOMAudioData.h b/content/events/src/nsDOMAudioData.h
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMAudioData.h
@@ -0,0 +1,24 @@
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsTArray.h"
+
+class nsDOMAudioData : public nsIDOMAudioData
+{
+public:
+ NS_DECL_ISUPPORTS
+ NS_DECL_NSIDOMAUDIODATA
+
+ nsDOMAudioData(nsTArray<float>& aData);
+
+ void Append(float aElement) { mData.AppendElement(aElement); }
+
+ float GetItemAt(PRUint32 aIndex)
+ {
+ return mData[aIndex];
+ }
+
+private:
+ ~nsDOMAudioData();
+
+protected:
+ nsTArray<float> mData;
+};
diff --git a/content/events/src/nsDOMEvent.cpp b/content/events/src/nsDOMEvent.cpp
--- a/content/events/src/nsDOMEvent.cpp
+++ b/content/events/src/nsDOMEvent.cpp
@@ -75,17 +75,17 @@ static const char* const sEventNames[] =
#ifdef MOZ_SVG
"SVGLoad", "SVGUnload", "SVGAbort", "SVGError", "SVGResize", "SVGScroll",
"SVGZoom",
#endif // MOZ_SVG
#ifdef MOZ_MEDIA
"loadstart", "progress", "suspend", "emptied", "stalled", "play", "pause",
"loadedmetadata", "loadeddata", "waiting", "playing", "canplay",
"canplaythrough", "seeking", "seeked", "timeupdate", "ended", "ratechange",
- "durationchange", "volumechange",
+ "durationchange", "volumechange", "audiowritten",
#endif // MOZ_MEDIA
"MozAfterPaint",
"MozSwipeGesture",
"MozMagnifyGestureStart",
"MozMagnifyGestureUpdate",
"MozMagnifyGesture",
"MozRotateGestureStart",
"MozRotateGestureUpdate",
@@ -678,16 +678,18 @@ nsDOMEvent::SetEventType(const nsAString
else if (atom == nsGkAtoms::onvolumechange)
mEvent->message = NS_VOLUMECHANGE;
else if (atom == nsGkAtoms::onload)
mEvent->message = NS_LOAD;
else if (atom == nsGkAtoms::onabort)
mEvent->message = NS_MEDIA_ABORT;
else if (atom == nsGkAtoms::onerror)
mEvent->message = NS_MEDIA_ERROR;
+ else if (atom == nsGkAtoms::onaudiowritten)
+ mEvent->message = NS_AUDIOWRITTEN;
}
#endif // MOZ_MEDIA
else if (mEvent->eventStructType == NS_SIMPLE_GESTURE_EVENT) {
if (atom == nsGkAtoms::onMozSwipeGesture)
mEvent->message = NS_SIMPLE_GESTURE_SWIPE;
else if (atom == nsGkAtoms::onMozMagnifyGestureStart)
mEvent->message = NS_SIMPLE_GESTURE_MAGNIFY_START;
else if (atom == nsGkAtoms::onMozMagnifyGestureUpdate)
@@ -1467,16 +1469,18 @@ const char* nsDOMEvent::GetEventName(PRU
case NS_ENDED:
return sEventNames[eDOMEvents_ended];
case NS_RATECHANGE:
return sEventNames[eDOMEvents_ratechange];
case NS_DURATIONCHANGE:
return sEventNames[eDOMEvents_durationchange];
case NS_VOLUMECHANGE:
return sEventNames[eDOMEvents_volumechange];
+ case NS_AUDIOWRITTEN:
+ return sEventNames[eDOMEvents_audiowritten];
#endif
case NS_AFTERPAINT:
return sEventNames[eDOMEvents_afterpaint];
case NS_SIMPLE_GESTURE_SWIPE:
return sEventNames[eDOMEvents_MozSwipeGesture];
case NS_SIMPLE_GESTURE_MAGNIFY_START:
return sEventNames[eDOMEvents_MozMagnifyGestureStart];
case NS_SIMPLE_GESTURE_MAGNIFY_UPDATE:
diff --git a/content/events/src/nsDOMEvent.h b/content/events/src/nsDOMEvent.h
--- a/content/events/src/nsDOMEvent.h
+++ b/content/events/src/nsDOMEvent.h
@@ -155,16 +155,17 @@ public:
eDOMEvents_canplaythrough,
eDOMEvents_seeking,
eDOMEvents_seeked,
eDOMEvents_timeupdate,
eDOMEvents_ended,
eDOMEvents_ratechange,
eDOMEvents_durationchange,
eDOMEvents_volumechange,
+ eDOMEvents_audiowritten,
#endif
eDOMEvents_afterpaint,
eDOMEvents_MozSwipeGesture,
eDOMEvents_MozMagnifyGestureStart,
eDOMEvents_MozMagnifyGestureUpdate,
eDOMEvents_MozMagnifyGesture,
eDOMEvents_MozRotateGestureStart,
eDOMEvents_MozRotateGestureUpdate,
diff --git a/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp b/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp
@@ -0,0 +1,68 @@
+#include "nsDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMAudioData.h"
+
+nsDOMNotifyAudioWrittenEvent::nsDOMNotifyAudioWrittenEvent(nsPresContext* aPresContext, nsEvent* aEvent,
+ PRUint32 aEventType, nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+ : nsDOMEvent(aPresContext, aEvent)
+{
+ if (mEvent) {
+ mEvent->message = aEventType;
+ }
+ mFrameBuffer = aFrameBuffer;
+ mSpectrum = aSpectrum;
+}
+
+NS_INTERFACE_MAP_BEGIN(nsDOMNotifyAudioWrittenEvent)
+ NS_INTERFACE_MAP_ENTRY(nsIDOMNotifyAudioWrittenEvent)
+ NS_INTERFACE_MAP_ENTRY_CONTENT_CLASSINFO(NotifyAudioWrittenEvent)
+NS_INTERFACE_MAP_END_INHERITING(nsDOMEvent)
+
+NS_IMPL_ADDREF_INHERITED(nsDOMNotifyAudioWrittenEvent, nsDOMEvent)
+NS_IMPL_RELEASE_INHERITED(nsDOMNotifyAudioWrittenEvent, nsDOMEvent)
+
+nsDOMNotifyAudioWrittenEvent::~nsDOMNotifyAudioWrittenEvent()
+{
+}
+
+NS_IMETHODIMP nsDOMNotifyAudioWrittenEvent::GetMozFrameBuffer(nsIDOMAudioData * *aResult)
+{
+ NS_IF_ADDREF(*aResult = mFrameBuffer);
+ return NS_OK;
+}
+
+NS_IMETHODIMP nsDOMNotifyAudioWrittenEvent::GetMozSpectrum(nsIDOMAudioData * *aResult)
+{
+ NS_IF_ADDREF(*aResult = mSpectrum);
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+nsDOMNotifyAudioWrittenEvent::InitAudioWrittenEvent(const nsAString& aType,
+ PRBool aCanBubble,
+ PRBool aCancelable,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+{
+ nsresult rv = nsDOMEvent::InitEvent(aType, aCanBubble, aCancelable);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ mFrameBuffer = aFrameBuffer;
+ mSpectrum = aSpectrum;
+ return NS_OK;
+}
+
+nsresult NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aInstancePtrResult,
+ nsPresContext* aPresContext,
+ nsEvent *aEvent,
+ PRUint32 aEventType,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+{
+ nsDOMNotifyAudioWrittenEvent* it = new nsDOMNotifyAudioWrittenEvent(aPresContext, aEvent, aEventType, aFrameBuffer, aSpectrum);
+ if (nsnull == it) {
+ return NS_ERROR_OUT_OF_MEMORY;
+ }
+
+ return CallQueryInterface(it, aInstancePtrResult);
+}
diff --git a/content/events/src/nsDOMNotifyAudioWrittenEvent.h b/content/events/src/nsDOMNotifyAudioWrittenEvent.h
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMNotifyAudioWrittenEvent.h
@@ -0,0 +1,35 @@
+#ifndef nsDOMNotifyAudioWrittenEvent_h_
+#define nsDOMNotifyAudioWrittenEvent_h_
+
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMEvent.h"
+#include "nsPresContext.h"
+
+class nsDOMNotifyAudioWrittenEvent : public nsDOMEvent,
+ public nsIDOMNotifyAudioWrittenEvent
+{
+public:
+ nsDOMNotifyAudioWrittenEvent(nsPresContext* aPresContext, nsEvent* aEvent,
+ PRUint32 aEventType, nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum);
+
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_NSIDOMNOTIFYAUDIOWRITTENEVENT
+
+ // Forward to base class
+ NS_FORWARD_TO_NSDOMEVENT
+
+nsresult NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aInstancePtrResult,
+ nsPresContext* aPresContext,
+ nsEvent *aEvent,
+ PRUint32 aEventType,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum);
+
+private:
+ ~nsDOMNotifyAudioWrittenEvent();
+ nsCOMPtr<nsIDOMAudioData> mFrameBuffer;
+ nsCOMPtr<nsIDOMAudioData> mSpectrum;
+};
+
+#endif // nsDOMNotifyAudioWrittenEvent_h_
diff --git a/content/events/src/nsEventDispatcher.cpp b/content/events/src/nsEventDispatcher.cpp
--- a/content/events/src/nsEventDispatcher.cpp
+++ b/content/events/src/nsEventDispatcher.cpp
@@ -768,11 +768,13 @@ nsEventDispatcher::CreateEvent(nsPresCon
if (aEventType.LowerCaseEqualsLiteral("simplegestureevent"))
return NS_NewDOMSimpleGestureEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("beforeunloadevent"))
return NS_NewDOMBeforeUnloadEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("pagetransition"))
return NS_NewDOMPageTransitionEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("scrollareaevent"))
return NS_NewDOMScrollAreaEvent(aDOMEvent, aPresContext, nsnull);
+ if (aEventType.LowerCaseEqualsLiteral("audiowrittenevent"))
+ return NS_NewDOMAudioWrittenEvent(aDOMEvent, aPresContext, nsnull);
return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
}
diff --git a/content/html/content/public/nsHTMLMediaElement.h b/content/html/content/public/nsHTMLMediaElement.h
--- a/content/html/content/public/nsHTMLMediaElement.h
+++ b/content/html/content/public/nsHTMLMediaElement.h
@@ -39,16 +39,18 @@
#include "nsGenericHTMLElement.h"
#include "nsMediaDecoder.h"
#include "nsIChannel.h"
#include "nsThreadUtils.h"
#include "nsIDOMRange.h"
#include "nsCycleCollectionParticipant.h"
#include "nsILoadGroup.h"
+#include "nsAudioStream.h"
+
// Define to output information on decoding and painting framerate
/* #define DEBUG_FRAME_RATE 1 */
typedef PRUint16 nsMediaNetworkState;
typedef PRUint16 nsMediaReadyState;
class nsHTMLMediaElement : public nsGenericHTMLElement
{
@@ -161,16 +163,17 @@ public:
gfxPattern::GraphicsFilter aFilter,
const gfxRect& aRect);
// Dispatch events
nsresult DispatchSimpleEvent(const nsAString& aName);
nsresult DispatchProgressEvent(const nsAString& aName);
nsresult DispatchAsyncSimpleEvent(const nsAString& aName);
nsresult DispatchAsyncProgressEvent(const nsAString& aName);
+ nsresult DispatchAudioWrittenEvent(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
// Called by the decoder when some data has been downloaded or
// buffering/seeking has ended. aNextFrameAvailable is true when
// the data for the next frame is available. This method will
// decide whether to set the ready state to HAVE_CURRENT_DATA,
// HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA.
enum NextFrameStatus {
// The next frame of audio/video is available
@@ -243,16 +246,21 @@ public:
void NotifyAddedSource();
/**
* Called when there's been an error fetching the resource. This decides
* whether it's appropriate to fire an error event.
*/
void NotifyLoadError();
+ /**
+ * Called when data has been written to the underlying audio stream.
+ */
+ void NotifyAudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
+
virtual PRBool IsNodeOfType(PRUint32 aFlags) const;
/**
* Returns the current load ID. Asynchronous events store the ID that was
* current when they were enqueued, and if it has changed when they come to
* fire, they consider themselves cancelled, and don't fire.
*/
PRUint32 GetCurrentLoadID() { return mCurrentLoadID; }
@@ -386,16 +394,23 @@ protected:
* Alias for Release(), but using stdcall calling convention so on
* platforms where Release has a strange calling convention (Windows)
* we can still get a method pointer to this method.
*/
void DoRelease() { Release(); }
nsRefPtr<nsMediaDecoder> mDecoder;
+ // XXX: just a hack to get an audio stream we can write to without an ogg/wave source.
+ nsAutoPtr<nsAudioStream> mAudioStream;
+
+ // XXX: just a hack to keep track of channels and rate info
+ PRUint32 mChannels;
+ PRUint32 mRate;
+
// Holds a reference to the first channel we open to the media resource.
// Once the decoder is created, control over the channel passes to the
// decoder, and we null out this reference. We must store this in case
// we need to cancel the channel before control of it passes to the decoder.
nsCOMPtr<nsIChannel> mChannel;
// Error attribute
nsCOMPtr<nsIDOMHTMLMediaError> mError;
diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp
--- a/content/html/content/src/nsHTMLMediaElement.cpp
+++ b/content/html/content/src/nsHTMLMediaElement.cpp
@@ -76,16 +76,21 @@
#include "nsIContentPolicy.h"
#include "nsContentPolicyUtils.h"
#include "nsContentErrors.h"
#include "nsCrossSiteListenerProxy.h"
#include "nsCycleCollectionParticipant.h"
#include "nsLayoutUtils.h"
#include "nsVideoFrame.h"
+#include "nsEventDispatcher.h"
+#include "nsIPrivateDOMEvent.h"
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMAudioData.h"
+
#ifdef MOZ_OGG
#include "nsOggDecoder.h"
#endif
#ifdef MOZ_WAVE
#include "nsWaveDecoder.h"
#endif
#ifdef PR_LOGGING
@@ -142,49 +147,64 @@ static PRLogModuleInfo* gMediaElementEve
class nsMediaEvent : public nsRunnable
{
public:
nsMediaEvent(nsHTMLMediaElement* aElement) :
mElement(aElement),
mLoadID(mElement->GetCurrentLoadID()) {}
+ nsMediaEvent(nsHTMLMediaElement* aElement, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ mElement(aElement),
+ mLoadID(mElement->GetCurrentLoadID()) {
+ mFrameBuffer.SwapElements(aFrameBuffer);
+ mSpectrum.SwapElements(aSpectrum);
+ }
~nsMediaEvent() {}
NS_IMETHOD Run() = 0;
+ nsTArray<float> mFrameBuffer;
+ nsTArray<float> mSpectrum;
protected:
PRBool IsCancelled() {
return mElement->GetCurrentLoadID() != mLoadID;
}
nsCOMPtr<nsHTMLMediaElement> mElement;
PRUint32 mLoadID;
};
-
class nsAsyncEventRunner : public nsMediaEvent
{
private:
nsString mName;
PRPackedBool mProgress;
public:
nsAsyncEventRunner(const nsAString& aName, nsHTMLMediaElement* aElement, PRBool aProgress) :
nsMediaEvent(aElement), mName(aName), mProgress(aProgress)
{
}
+ nsAsyncEventRunner(const nsAString& aName, nsHTMLMediaElement* aElement, PRBool aProgress, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ nsMediaEvent(aElement, aFrameBuffer, aSpectrum), mName(aName), mProgress(aProgress)
+ {
+ }
NS_IMETHOD Run() {
// Silently cancel if our load has been cancelled.
if (IsCancelled())
return NS_OK;
- return mProgress ?
- mElement->DispatchProgressEvent(mName) :
- mElement->DispatchSimpleEvent(mName);
+
+ if (mFrameBuffer.Length() > 0)
+ return mElement->DispatchAudioWrittenEvent(mFrameBuffer, mSpectrum);
+ else
+ return mProgress ?
+ mElement->DispatchProgressEvent(mName) :
+ mElement->DispatchSimpleEvent(mName);
}
};
class nsHTMLMediaElement::LoadNextSourceEvent : public nsMediaEvent {
public:
LoadNextSourceEvent(nsHTMLMediaElement *aElement)
: nsMediaEvent(aElement) {}
NS_IMETHOD Run() {
@@ -557,16 +577,27 @@ void nsHTMLMediaElement::NotifyLoadError
{
if (mIsLoadingFromSrcAttribute) {
NoSupportedMediaSourceError();
} else {
QueueLoadFromSourceTask();
}
}
+void nsHTMLMediaElement::NotifyAudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+// DispatchAsyncSimpleEvent(NS_LITERAL_STRING("audiowritten"));
+// LOG_EVENT(PR_LOG_DEBUG, ("%p Queuing simple event %s", this, NS_LITERAL_STRING("audiowritten").get()));
+
+ // XXX: do I need to dispatch to main thread agian?
+ nsCOMPtr<nsIRunnable> event =
+ new nsAsyncEventRunner(NS_LITERAL_STRING("audiowritten"), this, PR_FALSE, aFrameBuffer, aSpectrum);
+ NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+}
+
void nsHTMLMediaElement::LoadFromSourceChildren()
{
NS_ASSERTION(mDelayingLoadEvent,
"Should delay load event (if in document) during load");
while (PR_TRUE) {
nsresult rv;
nsCOMPtr<nsIURI> uri = GetNextSource();
if (!uri) {
@@ -717,16 +748,48 @@ NS_IMETHODIMP nsHTMLMediaElement::MozLoa
return rv;
}
DispatchAsyncProgressEvent(NS_LITERAL_STRING("loadstart"));
return NS_OK;
}
+NS_IMETHODIMP nsHTMLMediaElement::MozSetup(PRUint32 aChannels, PRUint32 aRate, float aVolume)
+{
+ mAudioStream = new nsAudioStream();
+ mAudioStream->Init(aChannels, aRate, nsAudioStream::FORMAT_FLOAT32);
+ // So we don't crash, in case Init doesn't work (params were wrong)...
+ if (mAudioStream) {
+ mChannels = aChannels;
+ mRate = aRate;
+ mAudioStream->SetVolume(aVolume);
+ return NS_OK;
+ } else
+ return NS_ERROR_FAILURE;
+}
+
+NS_IMETHODIMP nsHTMLMediaElement::MozWriteAudio(PRUint32 count, float *valueArray)
+{
+// // For now, assume there is a src and use it's stream
+// if (mDecoder)
+// mDecoder->Write(count, valueArray);
+
+ if (mAudioStream) {
+ // Make sure that we are going to write the correct amount of data based on number of channels
+ if (count % mChannels != 0)
+ return NS_ERROR_FAILURE;
+ else {
+ mAudioStream->Write(valueArray, count);
+ return NS_OK;
+ }
+ } else
+ return NS_ERROR_FAILURE;
+}
+
/* readonly attribute unsigned short readyState; */
NS_IMETHODIMP nsHTMLMediaElement::GetReadyState(PRUint16 *aReadyState)
{
*aReadyState = mReadyState;
return NS_OK;
}
@@ -894,17 +957,19 @@ nsHTMLMediaElement::nsHTMLMediaElement(n
mIsBindingToTree(PR_FALSE),
mIsRunningLoadMethod(PR_FALSE),
mIsLoadingFromSrcAttribute(PR_FALSE),
mDelayingLoadEvent(PR_FALSE),
mIsRunningSelectResource(PR_FALSE),
mSuspendedAfterFirstFrame(PR_FALSE),
mAllowSuspendAfterFirstFrame(PR_TRUE),
mHasPlayedOrSeeked(PR_FALSE),
- mHasSelfReference(PR_FALSE)
+ mHasSelfReference(PR_FALSE),
+ mChannels(0),
+ mRate(0)
{
#ifdef PR_LOGGING
if (!gMediaElementLog) {
gMediaElementLog = PR_NewLogModule("nsMediaElement");
}
if (!gMediaElementEventsLog) {
gMediaElementEventsLog = PR_NewLogModule("nsMediaElementEvents");
}
@@ -1727,21 +1792,43 @@ void nsHTMLMediaElement::Paint(gfxContex
aContext->NewPath();
aContext->PixelSnappedRectangleAndSetPattern(aRect, pat);
aContext->Fill();
} else if (mDecoder) {
mDecoder->Paint(aContext, aFilter, aRect);
}
}
+nsresult nsHTMLMediaElement::DispatchAudioWrittenEvent(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+ nsCOMPtr<nsIDOMDocumentEvent> docEvent(do_QueryInterface(GetOwnerDoc()));
+ nsCOMPtr<nsIDOMEventTarget> target(do_QueryInterface(static_cast<nsIContent*>(this)));
+ nsCOMPtr<nsIDOMEvent> event;
+
+ docEvent->CreateEvent(NS_LITERAL_STRING("AudioWrittenEvent"), getter_AddRefs(event));
+ nsCOMPtr<nsIDOMNotifyAudioWrittenEvent> audioWrittenEvent(do_QueryInterface(event));
+
+ nsRefPtr<nsDOMAudioData> frameBufferData = new nsDOMAudioData(aFrameBuffer);
+ if (!frameBufferData)
+ return NS_ERROR_OUT_OF_MEMORY;
+
+ nsRefPtr<nsDOMAudioData> spectrumData = new nsDOMAudioData(aSpectrum);
+ if (!spectrumData)
+ return NS_ERROR_OUT_OF_MEMORY;
+
+ audioWrittenEvent->InitAudioWrittenEvent(NS_LITERAL_STRING("audiowritten"), PR_TRUE, PR_TRUE, frameBufferData, spectrumData);
+
+ PRBool dummy;
+ return target->DispatchEvent(event, &dummy);
+}
+
nsresult nsHTMLMediaElement::DispatchSimpleEvent(const nsAString& aName)
{
LOG_EVENT(PR_LOG_DEBUG, ("%p Dispatching simple event %s", this,
NS_ConvertUTF16toUTF8(aName).get()));
-
return nsContentUtils::DispatchTrustedEvent(GetOwnerDoc(),
static_cast<nsIContent*>(this),
aName,
PR_TRUE,
PR_TRUE);
}
nsresult nsHTMLMediaElement::DispatchAsyncSimpleEvent(const nsAString& aName)
diff --git a/content/media/nsMediaDecoder.h b/content/media/nsMediaDecoder.h
--- a/content/media/nsMediaDecoder.h
+++ b/content/media/nsMediaDecoder.h
@@ -224,16 +224,19 @@ public:
// Moves any existing channel loads into the background, so that they don't
// block the load event. This is called when we stop delaying the load
// event. Any new loads initiated (for example to seek) will also be in the
// background. Implementations of this must call MoveLoadsToBackground() on
// their nsMediaStream.
virtual void MoveLoadsToBackground()=0;
+ // XXX: just a hack to test writing to audio stream
+ virtual void Write(PRUint32 count, float *valueArray)=0;
+
protected:
// Start timer to update download progress information.
nsresult StartProgress();
// Stop progress information timer.
nsresult StopProgress();
diff --git a/content/media/ogg/nsOggDecoder.cpp b/content/media/ogg/nsOggDecoder.cpp
--- a/content/media/ogg/nsOggDecoder.cpp
+++ b/content/media/ogg/nsOggDecoder.cpp
@@ -49,16 +49,21 @@
#include "nsChannelReader.h"
#include "nsHTMLVideoElement.h"
#include "nsIObserver.h"
#include "nsIObserverService.h"
#include "nsAutoLock.h"
#include "nsTArray.h"
#include "nsNetUtil.h"
+#include <math.h>
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
using mozilla::TimeDuration;
using mozilla::TimeStamp;
#ifdef PR_LOGGING
static PRLogModuleInfo* gOggDecoderLog;
#define LOG(type, msg) PR_LOG(gOggDecoderLog, type, msg)
#else
#define LOG(type, msg)
@@ -105,16 +110,19 @@ static PRLogModuleInfo* gOggDecoderLog;
#define BUFFERING_SECONDS_LOW_WATER_MARK 1
// The minimum size buffered byte range inside which we'll consider
// trying a bounded-seek. When we seek, we first try to seek inside all
// buffered ranges larger than this, and if they all fail we fall back to
// an unbounded seek over the whole media. 64K is approximately 16 pages.
#define MIN_BOUNDED_SEEK_SIZE (64 * 1024)
+// The size to use for the FFT lookup tables
+#define FFT_TABLE_SIZE 2048
+
class nsOggStepDecodeEvent;
/*
All reading (including seeks) from the nsMediaStream are done on the
decoding thread. The decoder thread is informed before closing that
the stream is about to close via the Shutdown
event. oggplay_prepare_for_close is called before sending the
shutdown event to tell liboggplay to shutdown.
@@ -176,17 +184,17 @@ public:
mVideoHeader = nsnull;
}
}
// Write the audio data from the frame to the Audio stream.
void Write(nsAudioStream* aStream)
{
aStream->Write(mAudioData.Elements(), mAudioData.Length());
- mAudioData.Clear();
+ mAudioData.Clear();
}
void SetVideoHeader(OggPlayDataHeader* aVideoHeader)
{
NS_ABORT_IF_FALSE(!mVideoHeader, "Frame already owns a video header");
mVideoHeader = aVideoHeader;
oggplay_callback_info_lock_item(mVideoHeader);
}
@@ -294,16 +302,18 @@ public:
// Cause state transitions. These methods obtain the decoder monitor
// to synchronise the change of state, and to notify other threads
// that the state has changed.
void Shutdown();
void Decode();
void Seek(float aTime);
void StopStepDecodeThread(nsAutoMonitor* aMonitor);
+ void Write(PRUint32 count, float *valueArray);
+
NS_IMETHOD Run();
PRBool HasAudio()
{
NS_ASSERTION(mState > DECODER_STATE_DECODING_METADATA, "HasAudio() called during invalid state");
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
return mAudioTrack != -1;
}
@@ -333,16 +343,22 @@ public:
// Play a frame of decoded video. The decode monitor is obtained
// internally by this method for synchronisation.
void PlayFrame();
// Play the video data from the given frame. The decode monitor
// must be locked when calling this method.
void PlayVideo(FrameData* aFrame);
+ // Calculate FFT for frame data
+ void CalculateFFT(nsTArray<float>& aData, nsTArray<float>& aSpectrum);
+
+ // Preload all FFT lookup tables
+ void PreloadTables();
+
// Plays the audio for the frame, plus any outstanding audio data
// buffered by nsAudioStream and not yet written to the
// hardware. The audio data for the frame is cleared out so
// subsequent calls with the same frame do not re-write the data.
// The decode monitor must be locked when calling this method.
void PlayAudio(FrameData* aFrame);
// Called from the main thread to get the current frame time. The decoder
@@ -613,16 +629,21 @@ private:
// Accessed by the step decode thread and the decode state machine thread.
// Synchronised via the decoder monitor.
PRPackedBool mBufferExhausted;
// PR_TRUE if mDuration has a value obtained from an HTTP header.
// Read/Written from the decode and main threads. Synchronised via the
// decoder monitor.
PRPackedBool mGotDurationFromHeader;
+
+ // FFT look-up tables
+ nsAutoTArray<PRUint32, FFT_TABLE_SIZE> mReverseTable;
+ nsAutoTArray<float, FFT_TABLE_SIZE> mCosTable;
+ nsAutoTArray<float, FFT_TABLE_SIZE> mSinTable;
};
// Event that gets posted to the thread that is responsible for decoding
// Ogg frames. Decodes each frame of an Ogg file. Locking of liboggplay
// is managed by liboggplay. The thread is created when the frames first
// need to be decoded and is shutdown when decoding is not needed (either
// completed, or seeking).
class nsOggStepDecodeEvent : public nsRunnable {
@@ -697,16 +718,61 @@ public:
}
}
mDecodeStateMachine->mDecodingCompleted = PR_TRUE;
return NS_OK;
}
};
+class nsAudioWrittenEventRunner : public nsRunnable
+{
+private:
+ nsCOMPtr<nsOggDecoder> mDecoder;
+ nsTArray<float> mFrameBuffer;
+ nsTArray<float> mSpectrum;
+
+public:
+ nsAudioWrittenEventRunner(nsOggDecoder* aDecoder, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ mDecoder(aDecoder)
+ {
+ mFrameBuffer.SwapElements(aFrameBuffer);
+ mSpectrum.SwapElements(aSpectrum);
+ }
+
+ NS_IMETHOD Run() {
+ mDecoder->AudioWritten(mFrameBuffer, mSpectrum);
+ return NS_OK;
+ }
+};
+
+
+// TODO: figure out where this should be done...
+void nsOggDecodeStateMachine::PreloadTables()
+{
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++)
+ mReverseTable.AppendElement(0);
+
+ PRUint32 limit = 1;
+ PRUint32 bit = FFT_TABLE_SIZE >> 1;
+
+ while (limit < FFT_TABLE_SIZE) {
+ for (PRUint32 i = 0; i < limit; i++)
+ mReverseTable[i + limit] = mReverseTable[i] + bit;
+
+ limit = limit << 1;
+ bit = bit >> 1;
+ }
+
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++) {
+ mSinTable.AppendElement(sin(-M_PI/i));
+ mCosTable.AppendElement(cos(-M_PI/i));
+ }
+}
+
nsOggDecodeStateMachine::nsOggDecodeStateMachine(nsOggDecoder* aDecoder) :
mDecoder(aDecoder),
mPlayer(0),
mPlayStartTime(),
mPauseStartTime(),
mPauseDuration(0),
mPlaying(PR_FALSE),
mCallbackPeriod(1.0),
@@ -728,16 +794,17 @@ nsOggDecodeStateMachine::nsOggDecodeStat
mDuration(-1),
mSeekable(PR_TRUE),
mPositionChangeQueued(PR_FALSE),
mDecodingCompleted(PR_FALSE),
mExitStepDecodeThread(PR_FALSE),
mBufferExhausted(PR_FALSE),
mGotDurationFromHeader(PR_FALSE)
{
+ PreloadTables();
}
nsOggDecodeStateMachine::~nsOggDecodeStateMachine()
{
while (!mDecodedFrames.IsEmpty()) {
delete mDecodedFrames.Pop();
}
oggplay_close(mPlayer);
@@ -1015,22 +1082,102 @@ void nsOggDecodeStateMachine::PlayVideo(
mDecoder->SetRGBData(aFrame->mVideoWidth, aFrame->mVideoHeight,
mFramerate, mAspectRatio, buffer.forget());
// Don't play the frame's video data more than once.
aFrame->ClearVideoHeader();
}
}
+void nsOggDecodeStateMachine::CalculateFFT(nsTArray<float>& aData, nsTArray<float>& aSpectrum)
+{
+ nsTArray<float> signal;
+ nsTArray<float> real;
+ nsTArray<float> imag;
+
+ // Assuming interlaced stereo channels, need to split and merge into a stero-mix mono signal
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++)
+ signal.AppendElement((aData[2*i] + aData[2*i+1]) / 2);
+
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++) {
+ real.AppendElement(signal[mReverseTable[i]]);
+ imag.AppendElement(0);
+ }
+
+ PRUint32 halfSize = 1;
+ float phaseShiftStepReal;
+ float phaseShiftStepImag;
+ float currentPhaseShiftReal;
+ float currentPhaseShiftImag;
+ PRUint32 i;
+ PRUint32 off;
+ float tr;
+ float ti;
+ float tmpReal;
+
+ while (halfSize < FFT_TABLE_SIZE) {
+ phaseShiftStepReal = mCosTable[halfSize];
+ phaseShiftStepImag = mSinTable[halfSize];
+ currentPhaseShiftReal = 1.0;
+ currentPhaseShiftImag = 0.0;
+
+ for (PRUint32 fftStep = 0; fftStep < halfSize; fftStep++) {
+ i = fftStep;
+
+ while (i < FFT_TABLE_SIZE) {
+ off = i + halfSize;
+ tr = (currentPhaseShiftReal * real[off]) - (currentPhaseShiftImag * imag[off]);
+ ti = (currentPhaseShiftReal * imag[off]) + (currentPhaseShiftImag * real[off]);
+
+ real[off] = real[i] - tr;
+ imag[off] = imag[i] - ti;
+ real[i] += tr;
+ imag[i] += ti;
+
+ i += halfSize << 1;
+ }
+
+ tmpReal = currentPhaseShiftReal;
+ currentPhaseShiftReal = (tmpReal * phaseShiftStepReal) - (currentPhaseShiftImag * phaseShiftStepImag);
+ currentPhaseShiftImag = (tmpReal * phaseShiftStepImag) + (currentPhaseShiftImag * phaseShiftStepReal);
+ }
+
+ halfSize = halfSize << 1;
+ }
+
+ // Calculate hypotenuse of the complex number with pythagorean theorem
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE / 2; i++) {
+ aSpectrum.AppendElement(sqrt(real[i] * real[i] + imag[i] * imag[i]) / FFT_TABLE_SIZE);
+ }
+}
+
void nsOggDecodeStateMachine::PlayAudio(FrameData* aFrame)
{
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
if (!mAudioStream)
return;
+ if (aFrame->mAudioData.Length() > 0) {
+ // Inform the element that we've written sound data.
+ nsTArray<float> spectrum;
+ nsTArray<float> frameBuffer = aFrame->mAudioData;
+
+ // Pad all frames to 4096 if they are less (some will be larger, that's OK).
+ if (frameBuffer.Length() < 4096) {
+ PRUint32 d = 4096 - frameBuffer.Length();
+ for (PRUint32 i = 0; i < d; i++)
+ frameBuffer.AppendElement(0);
+ }
+
+ CalculateFFT(frameBuffer, spectrum);
+
+ nsCOMPtr<nsIRunnable> event =
+ new nsAudioWrittenEventRunner(mDecoder, frameBuffer, spectrum);
+ NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+ }
aFrame->Write(mAudioStream);
}
void nsOggDecodeStateMachine::OpenAudioStream()
{
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
mAudioStream = new nsAudioStream();
if (!mAudioStream) {
@@ -1902,29 +2049,47 @@ void nsOggDecodeStateMachine::SetTracksA
}
if (mAudioTrack != -1 &&
oggplay_set_track_active(mPlayer, mAudioTrack) < 0) {
LOG(PR_LOG_ERROR, ("%p Could not set track %d active", mDecoder, mAudioTrack));
}
}
+void nsOggDecodeStateMachine::Write(PRUint32 count, float *valueArray)
+{
+ if (!mAudioStream) {
+ mAudioStream = new nsAudioStream();
+// mAudioStream->Init(mAudioChannels, mAudioRate, nsAudioStream::FORMAT_FLOAT32);
+ mAudioStream->Init(1, 44100, nsAudioStream::FORMAT_FLOAT32);
+// mAudioStream->SetVolume(mVolume);
+ mAudioStream->SetVolume(1);
+ }
+ mAudioStream->Write(valueArray, count);
+}
+
NS_IMPL_THREADSAFE_ISUPPORTS1(nsOggDecoder, nsIObserver)
void nsOggDecoder::Pause()
{
nsAutoMonitor mon(mMonitor);
if (mPlayState == PLAY_STATE_SEEKING || mPlayState == PLAY_STATE_ENDED) {
mNextState = PLAY_STATE_PAUSED;
return;
}
ChangeState(PLAY_STATE_PAUSED);
}
+void nsOggDecoder::Write(PRUint32 count, float *valueArray)
+{
+ if (mDecodeStateMachine)
+ mDecodeStateMachine->Write(count, valueArray);
+}
+
void nsOggDecoder::SetVolume(float volume)
{
nsAutoMonitor mon(mMonitor);
mInitialVolume = volume;
if (mDecodeStateMachine) {
mDecodeStateMachine->SetVolume(volume);
}
@@ -2135,16 +2300,24 @@ nsMediaStream* nsOggDecoder::GetCurrentS
already_AddRefed<nsIPrincipal> nsOggDecoder::GetCurrentPrincipal()
{
if (!mReader)
return nsnull;
return mReader->Stream()->GetCurrentPrincipal();
}
+ void nsOggDecoder::AudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+ if (mShuttingDown)
+ return;
+
+ mElement->NotifyAudioWritten(aFrameBuffer, aSpectrum);
+}
+
void nsOggDecoder::MetadataLoaded()
{
if (mShuttingDown)
return;
// Only inform the element of MetadataLoaded if not doing a load() in order
// to fulfill a seek, otherwise we'll get multiple metadataloaded events.
PRBool notifyElement = PR_TRUE;
diff --git a/content/media/ogg/nsOggDecoder.h b/content/media/ogg/nsOggDecoder.h
--- a/content/media/ogg/nsOggDecoder.h
+++ b/content/media/ogg/nsOggDecoder.h
@@ -325,16 +325,18 @@ class nsOggDecoder : public nsMediaDecod
virtual nsresult Seek(float time);
virtual nsresult PlaybackRateChanged();
virtual void Pause();
virtual void SetVolume(float volume);
virtual float GetDuration();
+ virtual void Write(PRUint32 count, float *valueArray);
+
virtual nsMediaStream* GetCurrentStream();
virtual already_AddRefed<nsIPrincipal> GetCurrentPrincipal();
virtual void NotifySuspendedStatusChanged();
virtual void NotifyBytesDownloaded();
virtual void NotifyDownloadEnded(nsresult aStatus);
// Called by nsChannelReader on the decoder thread
void NotifyBytesConsumed(PRInt64 aBytes);
@@ -383,16 +385,18 @@ class nsOggDecoder : public nsMediaDecod
// Tells our nsMediaStream to put all loads in the background.
virtual void MoveLoadsToBackground();
// Stop the state machine thread and drop references to the thread,
// state machine and channel reader.
void Stop();
+ void AudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
+
protected:
// Returns the monitor for other threads to synchronise access to
// state.
PRMonitor* GetMonitor()
{
return mMonitor;
}
diff --git a/content/media/wave/nsWaveDecoder.cpp b/content/media/wave/nsWaveDecoder.cpp
--- a/content/media/wave/nsWaveDecoder.cpp
+++ b/content/media/wave/nsWaveDecoder.cpp
@@ -1251,16 +1251,22 @@ void
nsWaveDecoder::Pause()
{
if (mPlaybackStateMachine) {
mPlaybackStateMachine->Pause();
}
}
void
+nsWaveDecoder::Write(PRUint32 count, float *valueArray)
+{
+ return;
+}
+
+void
nsWaveDecoder::SetVolume(float aVolume)
{
mInitialVolume = aVolume;
if (mPlaybackStateMachine) {
mPlaybackStateMachine->SetVolume(aVolume);
}
}
diff --git a/content/media/wave/nsWaveDecoder.h b/content/media/wave/nsWaveDecoder.h
--- a/content/media/wave/nsWaveDecoder.h
+++ b/content/media/wave/nsWaveDecoder.h
@@ -160,16 +160,18 @@ class nsWaveDecoder : public nsMediaDeco
virtual float GetDuration();
// Set the audio playback volume; must be in range [0.0, 1.0].
virtual void SetVolume(float aVolume);
virtual nsresult Play();
virtual void Pause();
+ virtual void Write(PRUint32 count, float *valueArray);
+
// Set the current time of the media to aTime. This may cause mStream to
// create a new channel to fetch data from the appropriate position in the
// stream.
virtual nsresult Seek(float aTime);
// Report whether the decoder is currently seeking.
virtual PRBool IsSeeking() const;
diff --git a/dom/base/nsDOMClassInfo.cpp b/dom/base/nsDOMClassInfo.cpp
--- a/dom/base/nsDOMClassInfo.cpp
+++ b/dom/base/nsDOMClassInfo.cpp
@@ -228,16 +228,17 @@
#include "nsIDOMBeforeUnloadEvent.h"
#include "nsIDOMMutationEvent.h"
#include "nsIDOMSmartCardEvent.h"
#include "nsIDOMXULCommandEvent.h"
#include "nsIDOMPageTransitionEvent.h"
#include "nsIDOMMessageEvent.h"
#include "nsPaintRequest.h"
#include "nsIDOMNotifyPaintEvent.h"
+#include "nsIDOMNotifyAudioWrittenEvent.h"
#include "nsIDOMScrollAreaEvent.h"
#include "nsIDOMNSDocumentStyle.h"
#include "nsIDOMDocumentRange.h"
#include "nsIDOMDocumentTraversal.h"
#include "nsIDOMDocumentXBL.h"
#include "nsIDOMDocumentView.h"
#include "nsIDOMElementCSSInlineStyle.h"
#include "nsIDOMLinkStyle.h"
@@ -1303,16 +1304,22 @@ static nsDOMClassInfoData sClassInfoData
// data transfer for drag and drop
NS_DEFINE_CLASSINFO_DATA(DataTransfer, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
NS_DEFINE_CLASSINFO_DATA(NotifyPaintEvent, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
+ NS_DEFINE_CLASSINFO_DATA(NotifyAudioWrittenEvent, nsDOMGenericSH,
+ DOM_DEFAULT_SCRIPTABLE_FLAGS)
+
+ NS_DEFINE_CLASSINFO_DATA(AudioData, nsDOMGenericSH,
+ DOM_DEFAULT_SCRIPTABLE_FLAGS)
+
NS_DEFINE_CLASSINFO_DATA(SimpleGestureEvent, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
#ifdef MOZ_MATHML
NS_DEFINE_CLASSINFO_DATA_WITH_NAME(MathMLElement, Element, nsElementSH,
ELEMENT_SCRIPTABLE_FLAGS)
#endif
@@ -3682,16 +3689,25 @@ nsDOMClassInfo::Init()
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNSDataTransfer)
DOM_CLASSINFO_MAP_END
DOM_CLASSINFO_MAP_BEGIN(NotifyPaintEvent, nsIDOMNotifyPaintEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNotifyPaintEvent)
DOM_CLASSINFO_EVENT_MAP_ENTRIES
DOM_CLASSINFO_MAP_END
+ DOM_CLASSINFO_MAP_BEGIN(NotifyAudioWrittenEvent, nsIDOMNotifyAudioWrittenEvent)
+ DOM_CLASSINFO_MAP_ENTRY(nsIDOMNotifyAudioWrittenEvent)
+ DOM_CLASSINFO_EVENT_MAP_ENTRIES
+ DOM_CLASSINFO_MAP_END
+
+ DOM_CLASSINFO_MAP_BEGIN(AudioData, nsIDOMAudioData)
+ DOM_CLASSINFO_MAP_ENTRY(nsIDOMAudioData)
+ DOM_CLASSINFO_MAP_END
+
DOM_CLASSINFO_MAP_BEGIN(SimpleGestureEvent, nsIDOMSimpleGestureEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMSimpleGestureEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMMouseEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNSMouseEvent)
DOM_CLASSINFO_UI_EVENT_MAP_ENTRIES
DOM_CLASSINFO_MAP_END
#ifdef MOZ_MATHML
diff --git a/dom/base/nsDOMClassInfoID.h b/dom/base/nsDOMClassInfoID.h
--- a/dom/base/nsDOMClassInfoID.h
+++ b/dom/base/nsDOMClassInfoID.h
@@ -450,16 +450,20 @@ enum nsDOMClassInfoID {
// DOM Traversal NodeIterator class
eDOMClassInfo_NodeIterator_id,
eDOMClassInfo_DataTransfer_id,
eDOMClassInfo_NotifyPaintEvent_id,
+ eDOMClassInfo_NotifyAudioWrittenEvent_id,
+
+ eDOMClassInfo_AudioData_id,
+
eDOMClassInfo_SimpleGestureEvent_id,
#ifdef MOZ_MATHML
eDOMClassInfo_MathMLElement_id,
#endif
eDOMClassInfo_Worker_id,
diff --git a/dom/interfaces/events/Makefile.in b/dom/interfaces/events/Makefile.in
--- a/dom/interfaces/events/Makefile.in
+++ b/dom/interfaces/events/Makefile.in
@@ -71,16 +71,17 @@ XPIDLSRCS = \
nsIDOMPopupBlockedEvent.idl \
nsIDOMBeforeUnloadEvent.idl \
nsIDOMNSEventTarget.idl \
nsIDOMSmartCardEvent.idl \
nsIDOMPageTransitionEvent.idl \
nsIDOMCommandEvent.idl \
nsIDOMMessageEvent.idl \
nsIDOMNotifyPaintEvent.idl \
+ nsIDOMNotifyAudioWrittenEvent.idl \
nsIDOMPaintRequest.idl \
nsIDOMPaintRequestList.idl \
nsIDOMSimpleGestureEvent.idl \
nsIDOMNSMouseEvent.idl \
nsIDOMOrientationEvent.idl \
nsIDOMScrollAreaEvent.idl \
$(NULL)
diff --git a/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl b/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl
new file mode 100644
--- /dev/null
+++ b/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl
@@ -0,0 +1,23 @@
+#include "nsIDOMEvent.idl"
+#include "domstubs.idl"
+
+[scriptable, uuid(9d4872bc-1aba-46c6-ad52-00ea927e08fb)]
+interface nsIDOMAudioData : nsISupports
+{
+ readonly attribute unsigned long length;
+ float item(in unsigned long index);
+};
+
+
+[scriptable, uuid(6250652d-7a6a-49a4-a2ee-9114e1e83427)]
+interface nsIDOMNotifyAudioWrittenEvent : nsIDOMEvent
+{
+ readonly attribute nsIDOMAudioData mozFrameBuffer;
+ readonly attribute nsIDOMAudioData mozSpectrum;
+
+ void initAudioWrittenEvent(in DOMString typeArg,
+ in boolean canBubbleArg,
+ in boolean cancelableArg,
+ in nsIDOMAudioData frameBuffer,
+ in nsIDOMAudioData spectrum);
+};
diff --git a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
--- a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
+++ b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
@@ -101,9 +101,15 @@ interface nsIDOMHTMLMediaElement : nsIDO
// Mozilla extension: load data from another media element. This is like
// load() but we don't run the resource selection algorithm; instead
// we just set our source to other's currentSrc. This is optimized
// so that this element will get access to all of other's cached/
// buffered data. In fact any future data downloaded by this element or
// other will be sharable by both elements.
void mozLoadFrom(in nsIDOMHTMLMediaElement other);
+
+ // Allow content to push audio data to the stream directly. Must call mozSetup first.
+ void mozWriteAudio(in PRUint32 count, [array, size_is(count)] in float valueArray);
+
+ // Setup the audio stream for writing
+ void mozSetup(in PRUint32 channels, in PRUint32 rate, in float volume);
};
diff --git a/widget/public/nsGUIEvent.h b/widget/public/nsGUIEvent.h
--- a/widget/public/nsGUIEvent.h
+++ b/widget/public/nsGUIEvent.h
@@ -395,18 +395,19 @@ class nsHashKey;
#define NS_CANPLAYTHROUGH (NS_MEDIA_EVENT_START+12)
#define NS_SEEKING (NS_MEDIA_EVENT_START+13)
#define NS_SEEKED (NS_MEDIA_EVENT_START+14)
#define NS_TIMEUPDATE (NS_MEDIA_EVENT_START+15)
#define NS_ENDED (NS_MEDIA_EVENT_START+16)
#define NS_RATECHANGE (NS_MEDIA_EVENT_START+17)
#define NS_DURATIONCHANGE (NS_MEDIA_EVENT_START+18)
#define NS_VOLUMECHANGE (NS_MEDIA_EVENT_START+19)
-#define NS_MEDIA_ABORT (NS_MEDIA_EVENT_START+20)
-#define NS_MEDIA_ERROR (NS_MEDIA_EVENT_START+21)
+#define NS_AUDIOWRITTEN (NS_MEDIA_EVENT_START+20)
+#define NS_MEDIA_ABORT (NS_MEDIA_EVENT_START+21)
+#define NS_MEDIA_ERROR (NS_MEDIA_EVENT_START+22)
#endif // MOZ_MEDIA
// paint notification events
#define NS_NOTIFYPAINT_START 3400
#define NS_AFTERPAINT (NS_NOTIFYPAINT_START)
// Simple gesture events
#define NS_SIMPLE_GESTURE_EVENT_START 3500
diff --git a/content/base/src/nsContentUtils.cpp b/content/base/src/nsContentUtils.cpp
--- a/content/base/src/nsContentUtils.cpp
+++ b/content/base/src/nsContentUtils.cpp
@@ -486,16 +486,17 @@ nsContentUtils::InitializeEventTable() {
{ &nsGkAtoms::oncanplaythrough, { NS_CANPLAYTHROUGH, EventNameType_HTML }},
{ &nsGkAtoms::onseeking, { NS_SEEKING, EventNameType_HTML }},
{ &nsGkAtoms::onseeked, { NS_SEEKED, EventNameType_HTML }},
{ &nsGkAtoms::ontimeupdate, { NS_TIMEUPDATE, EventNameType_HTML }},
{ &nsGkAtoms::onended, { NS_ENDED, EventNameType_HTML }},
{ &nsGkAtoms::onratechange, { NS_RATECHANGE, EventNameType_HTML }},
{ &nsGkAtoms::ondurationchange, { NS_DURATIONCHANGE, EventNameType_HTML }},
{ &nsGkAtoms::onvolumechange, { NS_VOLUMECHANGE, EventNameType_HTML }},
+ { &nsGkAtoms::onaudiowritten, { NS_AUDIOWRITTEN, EventNameType_HTML }},
#endif //MOZ_MEDIA
{ &nsGkAtoms::onMozAfterPaint, { NS_AFTERPAINT, EventNameType_None }},
{ &nsGkAtoms::onMozScrolledAreaChanged, { NS_SCROLLEDAREACHANGED, EventNameType_None }},
// Simple gesture events
{ &nsGkAtoms::onMozSwipeGesture, { NS_SIMPLE_GESTURE_SWIPE, EventNameType_None } },
{ &nsGkAtoms::onMozMagnifyGestureStart, { NS_SIMPLE_GESTURE_MAGNIFY_START, EventNameType_None } },
{ &nsGkAtoms::onMozMagnifyGestureUpdate, { NS_SIMPLE_GESTURE_MAGNIFY_UPDATE, EventNameType_None } },
diff --git a/content/base/src/nsGkAtomList.h b/content/base/src/nsGkAtomList.h
--- a/content/base/src/nsGkAtomList.h
+++ b/content/base/src/nsGkAtomList.h
@@ -1639,16 +1639,18 @@ GK_ATOM(seeking, "seeking")
GK_ATOM(seeked, "seeked")
GK_ATOM(timeupdate, "timeupdate")
GK_ATOM(ended, "ended")
GK_ATOM(canplay, "canplay")
GK_ATOM(canplaythrough, "canplaythrough")
GK_ATOM(ratechange, "ratechange")
GK_ATOM(durationchange, "durationchange")
GK_ATOM(volumechange, "volumechange")
+GK_ATOM(onaudiowritten, "onaudiowritten")
+GK_ATOM(audiowritten, "audiowritten")
#endif
// Frame property names
GK_ATOM(boxMetricsProperty, "BoxMetricsProperty") // nsBoxLayoutMetrics*
GK_ATOM(changeListProperty, "ChangeListProperty") // void*
GK_ATOM(collapseOffsetProperty, "CollapseOffsetProperty") // nsPoint*
GK_ATOM(computedOffsetProperty, "ComputedOffsetProperty") // nsPoint*
GK_ATOM(floatContinuationProperty, "FloatContinuationProperty") // nsFrameList*
diff --git a/content/events/public/nsIPrivateDOMEvent.h b/content/events/public/nsIPrivateDOMEvent.h
--- a/content/events/public/nsIPrivateDOMEvent.h
+++ b/content/events/public/nsIPrivateDOMEvent.h
@@ -45,16 +45,17 @@
{ 0xb0, 0x49, 0xdf, 0x9f, 0xc3, 0x6b, 0x56, 0xd4 } }
class nsIDOMEventTarget;
class nsIDOMEvent;
class nsEvent;
class nsCommandEvent;
class nsPresContext;
class nsInvalidateRequestList;
+class nsIDOMAudioData;
class nsIPrivateDOMEvent : public nsISupports
{
public:
NS_DECLARE_STATIC_IID_ACCESSOR(NS_IPRIVATEDOMEVENT_IID)
NS_IMETHOD DuplicatePrivateData() = 0;
NS_IMETHOD SetTarget(nsIDOMEventTarget* aTarget) = 0;
@@ -106,13 +107,21 @@ NS_NewDOMMessageEvent(nsIDOMEvent** aIns
nsresult
NS_NewDOMProgressEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsEvent* aEvent);
// This empties aInvalidateRequests.
nsresult
NS_NewDOMNotifyPaintEvent(nsIDOMEvent** aResult, nsPresContext* aPresContext,
nsEvent* aEvent,
PRUint32 aEventType = 0,
nsInvalidateRequestList* aInvalidateRequests = nsnull);
+
+nsresult
+NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aResult, nsPresContext* aPresContext,
+ nsEvent* aEvent,
+ PRUint32 aEventType = 0,
+ nsIDOMAudioData* aFrameBuffer = nsnull,
+ nsIDOMAudioData* aSpectrum = nsnull);
+
nsresult
NS_NewDOMSimpleGestureEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsSimpleGestureEvent* aEvent);
nsresult
NS_NewDOMScrollAreaEvent(nsIDOMEvent** aInstancePtrResult, nsPresContext* aPresContext, class nsScrollAreaEvent* aEvent);
#endif // nsIPrivateDOMEvent_h__
diff --git a/content/events/src/Makefile.in b/content/events/src/Makefile.in
--- a/content/events/src/Makefile.in
+++ b/content/events/src/Makefile.in
@@ -73,16 +73,18 @@ CPPSRCS = \
nsPLDOMEvent.cpp \
nsEventDispatcher.cpp \
nsIMEStateManager.cpp \
nsContentEventHandler.cpp \
nsEventListenerService.cpp \
nsDOMProgressEvent.cpp \
nsDOMDataTransfer.cpp \
nsDOMNotifyPaintEvent.cpp \
+ nsDOMNotifyAudioWrittenEvent.cpp \
+ nsDOMAudioData.cpp \
nsDOMSimpleGestureEvent.cpp \
nsDOMEventTargetHelper.cpp \
nsDOMScrollAreaEvent.cpp \
$(NULL)
# we don't want the shared lib, but we want to force the creation of a static lib.
FORCE_STATIC_LIB = 1
diff --git a/content/events/src/nsDOMAudioData.cpp b/content/events/src/nsDOMAudioData.cpp
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMAudioData.cpp
@@ -0,0 +1,34 @@
+#include "nsDOMAudioData.h"
+#include "nsCOMPtr.h"
+#include "nsContentUtils.h"
+
+NS_INTERFACE_TABLE_HEAD(nsDOMAudioData)
+ NS_INTERFACE_TABLE1(nsDOMAudioData, nsIDOMAudioData)
+ NS_INTERFACE_TABLE_TO_MAP_SEGUE
+ NS_INTERFACE_MAP_ENTRY_CONTENT_CLASSINFO(AudioData)
+NS_INTERFACE_MAP_END
+
+NS_IMPL_ADDREF(nsDOMAudioData)
+NS_IMPL_RELEASE(nsDOMAudioData)
+
+
+nsDOMAudioData::nsDOMAudioData(nsTArray<float>& aData)
+{
+ mData.SwapElements(aData);
+}
+
+nsDOMAudioData::~nsDOMAudioData()
+{
+}
+
+NS_IMETHODIMP nsDOMAudioData::GetLength(PRUint32 *aLength)
+{
+ *aLength = mData.Length();
+ return NS_OK;
+}
+
+NS_IMETHODIMP nsDOMAudioData::Item(PRUint32 aIndex, float *retval NS_OUTPARAM)
+{
+ *retval = GetItemAt(aIndex);
+ return NS_OK;
+}
diff --git a/content/events/src/nsDOMAudioData.h b/content/events/src/nsDOMAudioData.h
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMAudioData.h
@@ -0,0 +1,24 @@
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsTArray.h"
+
+class nsDOMAudioData : public nsIDOMAudioData
+{
+public:
+ NS_DECL_ISUPPORTS
+ NS_DECL_NSIDOMAUDIODATA
+
+ nsDOMAudioData(nsTArray<float>& aData);
+
+ void Append(float aElement) { mData.AppendElement(aElement); }
+
+ float GetItemAt(PRUint32 aIndex)
+ {
+ return mData[aIndex];
+ }
+
+private:
+ ~nsDOMAudioData();
+
+protected:
+ nsTArray<float> mData;
+};
diff --git a/content/events/src/nsDOMEvent.cpp b/content/events/src/nsDOMEvent.cpp
--- a/content/events/src/nsDOMEvent.cpp
+++ b/content/events/src/nsDOMEvent.cpp
@@ -75,17 +75,17 @@ static const char* const sEventNames[] =
#ifdef MOZ_SVG
"SVGLoad", "SVGUnload", "SVGAbort", "SVGError", "SVGResize", "SVGScroll",
"SVGZoom",
#endif // MOZ_SVG
#ifdef MOZ_MEDIA
"loadstart", "progress", "suspend", "emptied", "stalled", "play", "pause",
"loadedmetadata", "loadeddata", "waiting", "playing", "canplay",
"canplaythrough", "seeking", "seeked", "timeupdate", "ended", "ratechange",
- "durationchange", "volumechange",
+ "durationchange", "volumechange", "audiowritten",
#endif // MOZ_MEDIA
"MozAfterPaint",
"MozSwipeGesture",
"MozMagnifyGestureStart",
"MozMagnifyGestureUpdate",
"MozMagnifyGesture",
"MozRotateGestureStart",
"MozRotateGestureUpdate",
@@ -678,16 +678,18 @@ nsDOMEvent::SetEventType(const nsAString
else if (atom == nsGkAtoms::onvolumechange)
mEvent->message = NS_VOLUMECHANGE;
else if (atom == nsGkAtoms::onload)
mEvent->message = NS_LOAD;
else if (atom == nsGkAtoms::onabort)
mEvent->message = NS_MEDIA_ABORT;
else if (atom == nsGkAtoms::onerror)
mEvent->message = NS_MEDIA_ERROR;
+ else if (atom == nsGkAtoms::onaudiowritten)
+ mEvent->message = NS_AUDIOWRITTEN;
}
#endif // MOZ_MEDIA
else if (mEvent->eventStructType == NS_SIMPLE_GESTURE_EVENT) {
if (atom == nsGkAtoms::onMozSwipeGesture)
mEvent->message = NS_SIMPLE_GESTURE_SWIPE;
else if (atom == nsGkAtoms::onMozMagnifyGestureStart)
mEvent->message = NS_SIMPLE_GESTURE_MAGNIFY_START;
else if (atom == nsGkAtoms::onMozMagnifyGestureUpdate)
@@ -1467,16 +1469,18 @@ const char* nsDOMEvent::GetEventName(PRU
case NS_ENDED:
return sEventNames[eDOMEvents_ended];
case NS_RATECHANGE:
return sEventNames[eDOMEvents_ratechange];
case NS_DURATIONCHANGE:
return sEventNames[eDOMEvents_durationchange];
case NS_VOLUMECHANGE:
return sEventNames[eDOMEvents_volumechange];
+ case NS_AUDIOWRITTEN:
+ return sEventNames[eDOMEvents_audiowritten];
#endif
case NS_AFTERPAINT:
return sEventNames[eDOMEvents_afterpaint];
case NS_SIMPLE_GESTURE_SWIPE:
return sEventNames[eDOMEvents_MozSwipeGesture];
case NS_SIMPLE_GESTURE_MAGNIFY_START:
return sEventNames[eDOMEvents_MozMagnifyGestureStart];
case NS_SIMPLE_GESTURE_MAGNIFY_UPDATE:
diff --git a/content/events/src/nsDOMEvent.h b/content/events/src/nsDOMEvent.h
--- a/content/events/src/nsDOMEvent.h
+++ b/content/events/src/nsDOMEvent.h
@@ -155,16 +155,17 @@ public:
eDOMEvents_canplaythrough,
eDOMEvents_seeking,
eDOMEvents_seeked,
eDOMEvents_timeupdate,
eDOMEvents_ended,
eDOMEvents_ratechange,
eDOMEvents_durationchange,
eDOMEvents_volumechange,
+ eDOMEvents_audiowritten,
#endif
eDOMEvents_afterpaint,
eDOMEvents_MozSwipeGesture,
eDOMEvents_MozMagnifyGestureStart,
eDOMEvents_MozMagnifyGestureUpdate,
eDOMEvents_MozMagnifyGesture,
eDOMEvents_MozRotateGestureStart,
eDOMEvents_MozRotateGestureUpdate,
diff --git a/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp b/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMNotifyAudioWrittenEvent.cpp
@@ -0,0 +1,68 @@
+#include "nsDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMAudioData.h"
+
+nsDOMNotifyAudioWrittenEvent::nsDOMNotifyAudioWrittenEvent(nsPresContext* aPresContext, nsEvent* aEvent,
+ PRUint32 aEventType, nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+ : nsDOMEvent(aPresContext, aEvent)
+{
+ if (mEvent) {
+ mEvent->message = aEventType;
+ }
+ mFrameBuffer = aFrameBuffer;
+ mSpectrum = aSpectrum;
+}
+
+NS_INTERFACE_MAP_BEGIN(nsDOMNotifyAudioWrittenEvent)
+ NS_INTERFACE_MAP_ENTRY(nsIDOMNotifyAudioWrittenEvent)
+ NS_INTERFACE_MAP_ENTRY_CONTENT_CLASSINFO(NotifyAudioWrittenEvent)
+NS_INTERFACE_MAP_END_INHERITING(nsDOMEvent)
+
+NS_IMPL_ADDREF_INHERITED(nsDOMNotifyAudioWrittenEvent, nsDOMEvent)
+NS_IMPL_RELEASE_INHERITED(nsDOMNotifyAudioWrittenEvent, nsDOMEvent)
+
+nsDOMNotifyAudioWrittenEvent::~nsDOMNotifyAudioWrittenEvent()
+{
+}
+
+NS_IMETHODIMP nsDOMNotifyAudioWrittenEvent::GetMozFrameBuffer(nsIDOMAudioData * *aResult)
+{
+ NS_IF_ADDREF(*aResult = mFrameBuffer);
+ return NS_OK;
+}
+
+NS_IMETHODIMP nsDOMNotifyAudioWrittenEvent::GetMozSpectrum(nsIDOMAudioData * *aResult)
+{
+ NS_IF_ADDREF(*aResult = mSpectrum);
+ return NS_OK;
+}
+
+NS_IMETHODIMP
+nsDOMNotifyAudioWrittenEvent::InitAudioWrittenEvent(const nsAString& aType,
+ PRBool aCanBubble,
+ PRBool aCancelable,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+{
+ nsresult rv = nsDOMEvent::InitEvent(aType, aCanBubble, aCancelable);
+ NS_ENSURE_SUCCESS(rv, rv);
+
+ mFrameBuffer = aFrameBuffer;
+ mSpectrum = aSpectrum;
+ return NS_OK;
+}
+
+nsresult NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aInstancePtrResult,
+ nsPresContext* aPresContext,
+ nsEvent *aEvent,
+ PRUint32 aEventType,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum)
+{
+ nsDOMNotifyAudioWrittenEvent* it = new nsDOMNotifyAudioWrittenEvent(aPresContext, aEvent, aEventType, aFrameBuffer, aSpectrum);
+ if (nsnull == it) {
+ return NS_ERROR_OUT_OF_MEMORY;
+ }
+
+ return CallQueryInterface(it, aInstancePtrResult);
+}
diff --git a/content/events/src/nsDOMNotifyAudioWrittenEvent.h b/content/events/src/nsDOMNotifyAudioWrittenEvent.h
new file mode 100644
--- /dev/null
+++ b/content/events/src/nsDOMNotifyAudioWrittenEvent.h
@@ -0,0 +1,35 @@
+#ifndef nsDOMNotifyAudioWrittenEvent_h_
+#define nsDOMNotifyAudioWrittenEvent_h_
+
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMEvent.h"
+#include "nsPresContext.h"
+
+class nsDOMNotifyAudioWrittenEvent : public nsDOMEvent,
+ public nsIDOMNotifyAudioWrittenEvent
+{
+public:
+ nsDOMNotifyAudioWrittenEvent(nsPresContext* aPresContext, nsEvent* aEvent,
+ PRUint32 aEventType, nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum);
+
+ NS_DECL_ISUPPORTS_INHERITED
+ NS_DECL_NSIDOMNOTIFYAUDIOWRITTENEVENT
+
+ // Forward to base class
+ NS_FORWARD_TO_NSDOMEVENT
+
+nsresult NS_NewDOMAudioWrittenEvent(nsIDOMEvent** aInstancePtrResult,
+ nsPresContext* aPresContext,
+ nsEvent *aEvent,
+ PRUint32 aEventType,
+ nsIDOMAudioData* aFrameBuffer,
+ nsIDOMAudioData* aSpectrum);
+
+private:
+ ~nsDOMNotifyAudioWrittenEvent();
+ nsCOMPtr<nsIDOMAudioData> mFrameBuffer;
+ nsCOMPtr<nsIDOMAudioData> mSpectrum;
+};
+
+#endif // nsDOMNotifyAudioWrittenEvent_h_
diff --git a/content/events/src/nsEventDispatcher.cpp b/content/events/src/nsEventDispatcher.cpp
--- a/content/events/src/nsEventDispatcher.cpp
+++ b/content/events/src/nsEventDispatcher.cpp
@@ -768,11 +768,13 @@ nsEventDispatcher::CreateEvent(nsPresCon
if (aEventType.LowerCaseEqualsLiteral("simplegestureevent"))
return NS_NewDOMSimpleGestureEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("beforeunloadevent"))
return NS_NewDOMBeforeUnloadEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("pagetransition"))
return NS_NewDOMPageTransitionEvent(aDOMEvent, aPresContext, nsnull);
if (aEventType.LowerCaseEqualsLiteral("scrollareaevent"))
return NS_NewDOMScrollAreaEvent(aDOMEvent, aPresContext, nsnull);
+ if (aEventType.LowerCaseEqualsLiteral("audiowrittenevent"))
+ return NS_NewDOMAudioWrittenEvent(aDOMEvent, aPresContext, nsnull);
return NS_ERROR_DOM_NOT_SUPPORTED_ERR;
}
diff --git a/content/html/content/public/nsHTMLMediaElement.h b/content/html/content/public/nsHTMLMediaElement.h
--- a/content/html/content/public/nsHTMLMediaElement.h
+++ b/content/html/content/public/nsHTMLMediaElement.h
@@ -39,16 +39,18 @@
#include "nsGenericHTMLElement.h"
#include "nsMediaDecoder.h"
#include "nsIChannel.h"
#include "nsThreadUtils.h"
#include "nsIDOMRange.h"
#include "nsCycleCollectionParticipant.h"
#include "nsILoadGroup.h"
+#include "nsAudioStream.h"
+
// Define to output information on decoding and painting framerate
/* #define DEBUG_FRAME_RATE 1 */
typedef PRUint16 nsMediaNetworkState;
typedef PRUint16 nsMediaReadyState;
class nsHTMLMediaElement : public nsGenericHTMLElement
{
@@ -161,16 +163,17 @@ public:
gfxPattern::GraphicsFilter aFilter,
const gfxRect& aRect);
// Dispatch events
nsresult DispatchSimpleEvent(const nsAString& aName);
nsresult DispatchProgressEvent(const nsAString& aName);
nsresult DispatchAsyncSimpleEvent(const nsAString& aName);
nsresult DispatchAsyncProgressEvent(const nsAString& aName);
+ nsresult DispatchAudioWrittenEvent(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
// Called by the decoder when some data has been downloaded or
// buffering/seeking has ended. aNextFrameAvailable is true when
// the data for the next frame is available. This method will
// decide whether to set the ready state to HAVE_CURRENT_DATA,
// HAVE_FUTURE_DATA or HAVE_ENOUGH_DATA.
enum NextFrameStatus {
// The next frame of audio/video is available
@@ -243,16 +246,21 @@ public:
void NotifyAddedSource();
/**
* Called when there's been an error fetching the resource. This decides
* whether it's appropriate to fire an error event.
*/
void NotifyLoadError();
+ /**
+ * Called when data has been written to the underlying audio stream.
+ */
+ void NotifyAudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
+
virtual PRBool IsNodeOfType(PRUint32 aFlags) const;
/**
* Returns the current load ID. Asynchronous events store the ID that was
* current when they were enqueued, and if it has changed when they come to
* fire, they consider themselves cancelled, and don't fire.
*/
PRUint32 GetCurrentLoadID() { return mCurrentLoadID; }
@@ -386,16 +394,23 @@ protected:
* Alias for Release(), but using stdcall calling convention so on
* platforms where Release has a strange calling convention (Windows)
* we can still get a method pointer to this method.
*/
void DoRelease() { Release(); }
nsRefPtr<nsMediaDecoder> mDecoder;
+ // XXX: just a hack to get an audio stream we can write to without an ogg/wave source.
+ nsAutoPtr<nsAudioStream> mAudioStream;
+
+ // XXX: just a hack to keep track of channels and rate info
+ PRUint32 mChannels;
+ PRUint32 mRate;
+
// Holds a reference to the first channel we open to the media resource.
// Once the decoder is created, control over the channel passes to the
// decoder, and we null out this reference. We must store this in case
// we need to cancel the channel before control of it passes to the decoder.
nsCOMPtr<nsIChannel> mChannel;
// Error attribute
nsCOMPtr<nsIDOMHTMLMediaError> mError;
diff --git a/content/html/content/src/nsHTMLMediaElement.cpp b/content/html/content/src/nsHTMLMediaElement.cpp
--- a/content/html/content/src/nsHTMLMediaElement.cpp
+++ b/content/html/content/src/nsHTMLMediaElement.cpp
@@ -76,16 +76,21 @@
#include "nsIContentPolicy.h"
#include "nsContentPolicyUtils.h"
#include "nsContentErrors.h"
#include "nsCrossSiteListenerProxy.h"
#include "nsCycleCollectionParticipant.h"
#include "nsLayoutUtils.h"
#include "nsVideoFrame.h"
+#include "nsEventDispatcher.h"
+#include "nsIPrivateDOMEvent.h"
+#include "nsIDOMNotifyAudioWrittenEvent.h"
+#include "nsDOMAudioData.h"
+
#ifdef MOZ_OGG
#include "nsOggDecoder.h"
#endif
#ifdef MOZ_WAVE
#include "nsWaveDecoder.h"
#endif
#ifdef PR_LOGGING
@@ -142,49 +147,64 @@ static PRLogModuleInfo* gMediaElementEve
class nsMediaEvent : public nsRunnable
{
public:
nsMediaEvent(nsHTMLMediaElement* aElement) :
mElement(aElement),
mLoadID(mElement->GetCurrentLoadID()) {}
+ nsMediaEvent(nsHTMLMediaElement* aElement, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ mElement(aElement),
+ mLoadID(mElement->GetCurrentLoadID()) {
+ mFrameBuffer.SwapElements(aFrameBuffer);
+ mSpectrum.SwapElements(aSpectrum);
+ }
~nsMediaEvent() {}
NS_IMETHOD Run() = 0;
+ nsTArray<float> mFrameBuffer;
+ nsTArray<float> mSpectrum;
protected:
PRBool IsCancelled() {
return mElement->GetCurrentLoadID() != mLoadID;
}
nsCOMPtr<nsHTMLMediaElement> mElement;
PRUint32 mLoadID;
};
-
class nsAsyncEventRunner : public nsMediaEvent
{
private:
nsString mName;
PRPackedBool mProgress;
public:
nsAsyncEventRunner(const nsAString& aName, nsHTMLMediaElement* aElement, PRBool aProgress) :
nsMediaEvent(aElement), mName(aName), mProgress(aProgress)
{
}
+ nsAsyncEventRunner(const nsAString& aName, nsHTMLMediaElement* aElement, PRBool aProgress, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ nsMediaEvent(aElement, aFrameBuffer, aSpectrum), mName(aName), mProgress(aProgress)
+ {
+ }
NS_IMETHOD Run() {
// Silently cancel if our load has been cancelled.
if (IsCancelled())
return NS_OK;
- return mProgress ?
- mElement->DispatchProgressEvent(mName) :
- mElement->DispatchSimpleEvent(mName);
+
+ if (mFrameBuffer.Length() > 0)
+ return mElement->DispatchAudioWrittenEvent(mFrameBuffer, mSpectrum);
+ else
+ return mProgress ?
+ mElement->DispatchProgressEvent(mName) :
+ mElement->DispatchSimpleEvent(mName);
}
};
class nsHTMLMediaElement::LoadNextSourceEvent : public nsMediaEvent {
public:
LoadNextSourceEvent(nsHTMLMediaElement *aElement)
: nsMediaEvent(aElement) {}
NS_IMETHOD Run() {
@@ -557,16 +577,27 @@ void nsHTMLMediaElement::NotifyLoadError
{
if (mIsLoadingFromSrcAttribute) {
NoSupportedMediaSourceError();
} else {
QueueLoadFromSourceTask();
}
}
+void nsHTMLMediaElement::NotifyAudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+// DispatchAsyncSimpleEvent(NS_LITERAL_STRING("audiowritten"));
+// LOG_EVENT(PR_LOG_DEBUG, ("%p Queuing simple event %s", this, NS_LITERAL_STRING("audiowritten").get()));
+
+ // XXX: do I need to dispatch to main thread agian?
+ nsCOMPtr<nsIRunnable> event =
+ new nsAsyncEventRunner(NS_LITERAL_STRING("audiowritten"), this, PR_FALSE, aFrameBuffer, aSpectrum);
+ NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+}
+
void nsHTMLMediaElement::LoadFromSourceChildren()
{
NS_ASSERTION(mDelayingLoadEvent,
"Should delay load event (if in document) during load");
while (PR_TRUE) {
nsresult rv;
nsCOMPtr<nsIURI> uri = GetNextSource();
if (!uri) {
@@ -717,16 +748,48 @@ NS_IMETHODIMP nsHTMLMediaElement::MozLoa
return rv;
}
DispatchAsyncProgressEvent(NS_LITERAL_STRING("loadstart"));
return NS_OK;
}
+NS_IMETHODIMP nsHTMLMediaElement::MozSetup(PRUint32 aChannels, PRUint32 aRate, float aVolume)
+{
+ mAudioStream = new nsAudioStream();
+ mAudioStream->Init(aChannels, aRate, nsAudioStream::FORMAT_FLOAT32);
+ // So we don't crash, in case Init doesn't work (params were wrong)...
+ if (mAudioStream) {
+ mChannels = aChannels;
+ mRate = aRate;
+ mAudioStream->SetVolume(aVolume);
+ return NS_OK;
+ } else
+ return NS_ERROR_FAILURE;
+}
+
+NS_IMETHODIMP nsHTMLMediaElement::MozWriteAudio(PRUint32 count, float *valueArray)
+{
+// // For now, assume there is a src and use it's stream
+// if (mDecoder)
+// mDecoder->Write(count, valueArray);
+
+ if (mAudioStream) {
+ // Make sure that we are going to write the correct amount of data based on number of channels
+ if (count % mChannels != 0)
+ return NS_ERROR_FAILURE;
+ else {
+ mAudioStream->Write(valueArray, count);
+ return NS_OK;
+ }
+ } else
+ return NS_ERROR_FAILURE;
+}
+
/* readonly attribute unsigned short readyState; */
NS_IMETHODIMP nsHTMLMediaElement::GetReadyState(PRUint16 *aReadyState)
{
*aReadyState = mReadyState;
return NS_OK;
}
@@ -894,17 +957,19 @@ nsHTMLMediaElement::nsHTMLMediaElement(n
mIsBindingToTree(PR_FALSE),
mIsRunningLoadMethod(PR_FALSE),
mIsLoadingFromSrcAttribute(PR_FALSE),
mDelayingLoadEvent(PR_FALSE),
mIsRunningSelectResource(PR_FALSE),
mSuspendedAfterFirstFrame(PR_FALSE),
mAllowSuspendAfterFirstFrame(PR_TRUE),
mHasPlayedOrSeeked(PR_FALSE),
- mHasSelfReference(PR_FALSE)
+ mHasSelfReference(PR_FALSE),
+ mChannels(0),
+ mRate(0)
{
#ifdef PR_LOGGING
if (!gMediaElementLog) {
gMediaElementLog = PR_NewLogModule("nsMediaElement");
}
if (!gMediaElementEventsLog) {
gMediaElementEventsLog = PR_NewLogModule("nsMediaElementEvents");
}
@@ -1727,21 +1792,43 @@ void nsHTMLMediaElement::Paint(gfxContex
aContext->NewPath();
aContext->PixelSnappedRectangleAndSetPattern(aRect, pat);
aContext->Fill();
} else if (mDecoder) {
mDecoder->Paint(aContext, aFilter, aRect);
}
}
+nsresult nsHTMLMediaElement::DispatchAudioWrittenEvent(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+ nsCOMPtr<nsIDOMDocumentEvent> docEvent(do_QueryInterface(GetOwnerDoc()));
+ nsCOMPtr<nsIDOMEventTarget> target(do_QueryInterface(static_cast<nsIContent*>(this)));
+ nsCOMPtr<nsIDOMEvent> event;
+
+ docEvent->CreateEvent(NS_LITERAL_STRING("AudioWrittenEvent"), getter_AddRefs(event));
+ nsCOMPtr<nsIDOMNotifyAudioWrittenEvent> audioWrittenEvent(do_QueryInterface(event));
+
+ nsRefPtr<nsDOMAudioData> frameBufferData = new nsDOMAudioData(aFrameBuffer);
+ if (!frameBufferData)
+ return NS_ERROR_OUT_OF_MEMORY;
+
+ nsRefPtr<nsDOMAudioData> spectrumData = new nsDOMAudioData(aSpectrum);
+ if (!spectrumData)
+ return NS_ERROR_OUT_OF_MEMORY;
+
+ audioWrittenEvent->InitAudioWrittenEvent(NS_LITERAL_STRING("audiowritten"), PR_TRUE, PR_TRUE, frameBufferData, spectrumData);
+
+ PRBool dummy;
+ return target->DispatchEvent(event, &dummy);
+}
+
nsresult nsHTMLMediaElement::DispatchSimpleEvent(const nsAString& aName)
{
LOG_EVENT(PR_LOG_DEBUG, ("%p Dispatching simple event %s", this,
NS_ConvertUTF16toUTF8(aName).get()));
-
return nsContentUtils::DispatchTrustedEvent(GetOwnerDoc(),
static_cast<nsIContent*>(this),
aName,
PR_TRUE,
PR_TRUE);
}
nsresult nsHTMLMediaElement::DispatchAsyncSimpleEvent(const nsAString& aName)
diff --git a/content/media/nsMediaDecoder.h b/content/media/nsMediaDecoder.h
--- a/content/media/nsMediaDecoder.h
+++ b/content/media/nsMediaDecoder.h
@@ -224,16 +224,19 @@ public:
// Moves any existing channel loads into the background, so that they don't
// block the load event. This is called when we stop delaying the load
// event. Any new loads initiated (for example to seek) will also be in the
// background. Implementations of this must call MoveLoadsToBackground() on
// their nsMediaStream.
virtual void MoveLoadsToBackground()=0;
+ // XXX: just a hack to test writing to audio stream
+ virtual void Write(PRUint32 count, float *valueArray)=0;
+
protected:
// Start timer to update download progress information.
nsresult StartProgress();
// Stop progress information timer.
nsresult StopProgress();
diff --git a/content/media/ogg/nsOggDecoder.cpp b/content/media/ogg/nsOggDecoder.cpp
--- a/content/media/ogg/nsOggDecoder.cpp
+++ b/content/media/ogg/nsOggDecoder.cpp
@@ -49,16 +49,21 @@
#include "nsChannelReader.h"
#include "nsHTMLVideoElement.h"
#include "nsIObserver.h"
#include "nsIObserverService.h"
#include "nsAutoLock.h"
#include "nsTArray.h"
#include "nsNetUtil.h"
+#include <math.h>
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
using mozilla::TimeDuration;
using mozilla::TimeStamp;
#ifdef PR_LOGGING
static PRLogModuleInfo* gOggDecoderLog;
#define LOG(type, msg) PR_LOG(gOggDecoderLog, type, msg)
#else
#define LOG(type, msg)
@@ -105,16 +110,19 @@ static PRLogModuleInfo* gOggDecoderLog;
#define BUFFERING_SECONDS_LOW_WATER_MARK 1
// The minimum size buffered byte range inside which we'll consider
// trying a bounded-seek. When we seek, we first try to seek inside all
// buffered ranges larger than this, and if they all fail we fall back to
// an unbounded seek over the whole media. 64K is approximately 16 pages.
#define MIN_BOUNDED_SEEK_SIZE (64 * 1024)
+// The size to use for the FFT lookup tables
+#define FFT_TABLE_SIZE 2048
+
class nsOggStepDecodeEvent;
/*
All reading (including seeks) from the nsMediaStream are done on the
decoding thread. The decoder thread is informed before closing that
the stream is about to close via the Shutdown
event. oggplay_prepare_for_close is called before sending the
shutdown event to tell liboggplay to shutdown.
@@ -176,17 +184,17 @@ public:
mVideoHeader = nsnull;
}
}
// Write the audio data from the frame to the Audio stream.
void Write(nsAudioStream* aStream)
{
aStream->Write(mAudioData.Elements(), mAudioData.Length());
- mAudioData.Clear();
+ mAudioData.Clear();
}
void SetVideoHeader(OggPlayDataHeader* aVideoHeader)
{
NS_ABORT_IF_FALSE(!mVideoHeader, "Frame already owns a video header");
mVideoHeader = aVideoHeader;
oggplay_callback_info_lock_item(mVideoHeader);
}
@@ -294,16 +302,18 @@ public:
// Cause state transitions. These methods obtain the decoder monitor
// to synchronise the change of state, and to notify other threads
// that the state has changed.
void Shutdown();
void Decode();
void Seek(float aTime);
void StopStepDecodeThread(nsAutoMonitor* aMonitor);
+ void Write(PRUint32 count, float *valueArray);
+
NS_IMETHOD Run();
PRBool HasAudio()
{
NS_ASSERTION(mState > DECODER_STATE_DECODING_METADATA, "HasAudio() called during invalid state");
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
return mAudioTrack != -1;
}
@@ -333,16 +343,22 @@ public:
// Play a frame of decoded video. The decode monitor is obtained
// internally by this method for synchronisation.
void PlayFrame();
// Play the video data from the given frame. The decode monitor
// must be locked when calling this method.
void PlayVideo(FrameData* aFrame);
+ // Calculate FFT for frame data
+ void CalculateFFT(nsTArray<float>& aData, nsTArray<float>& aSpectrum);
+
+ // Preload all FFT lookup tables
+ void PreloadTables();
+
// Plays the audio for the frame, plus any outstanding audio data
// buffered by nsAudioStream and not yet written to the
// hardware. The audio data for the frame is cleared out so
// subsequent calls with the same frame do not re-write the data.
// The decode monitor must be locked when calling this method.
void PlayAudio(FrameData* aFrame);
// Called from the main thread to get the current frame time. The decoder
@@ -613,16 +629,21 @@ private:
// Accessed by the step decode thread and the decode state machine thread.
// Synchronised via the decoder monitor.
PRPackedBool mBufferExhausted;
// PR_TRUE if mDuration has a value obtained from an HTTP header.
// Read/Written from the decode and main threads. Synchronised via the
// decoder monitor.
PRPackedBool mGotDurationFromHeader;
+
+ // FFT look-up tables
+ nsAutoTArray<PRUint32, FFT_TABLE_SIZE> mReverseTable;
+ nsAutoTArray<float, FFT_TABLE_SIZE> mCosTable;
+ nsAutoTArray<float, FFT_TABLE_SIZE> mSinTable;
};
// Event that gets posted to the thread that is responsible for decoding
// Ogg frames. Decodes each frame of an Ogg file. Locking of liboggplay
// is managed by liboggplay. The thread is created when the frames first
// need to be decoded and is shutdown when decoding is not needed (either
// completed, or seeking).
class nsOggStepDecodeEvent : public nsRunnable {
@@ -697,16 +718,61 @@ public:
}
}
mDecodeStateMachine->mDecodingCompleted = PR_TRUE;
return NS_OK;
}
};
+class nsAudioWrittenEventRunner : public nsRunnable
+{
+private:
+ nsCOMPtr<nsOggDecoder> mDecoder;
+ nsTArray<float> mFrameBuffer;
+ nsTArray<float> mSpectrum;
+
+public:
+ nsAudioWrittenEventRunner(nsOggDecoder* aDecoder, nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum) :
+ mDecoder(aDecoder)
+ {
+ mFrameBuffer.SwapElements(aFrameBuffer);
+ mSpectrum.SwapElements(aSpectrum);
+ }
+
+ NS_IMETHOD Run() {
+ mDecoder->AudioWritten(mFrameBuffer, mSpectrum);
+ return NS_OK;
+ }
+};
+
+
+// TODO: figure out where this should be done...
+void nsOggDecodeStateMachine::PreloadTables()
+{
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++)
+ mReverseTable.AppendElement(0);
+
+ PRUint32 limit = 1;
+ PRUint32 bit = FFT_TABLE_SIZE >> 1;
+
+ while (limit < FFT_TABLE_SIZE) {
+ for (PRUint32 i = 0; i < limit; i++)
+ mReverseTable[i + limit] = mReverseTable[i] + bit;
+
+ limit = limit << 1;
+ bit = bit >> 1;
+ }
+
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++) {
+ mSinTable.AppendElement(sin(-M_PI/i));
+ mCosTable.AppendElement(cos(-M_PI/i));
+ }
+}
+
nsOggDecodeStateMachine::nsOggDecodeStateMachine(nsOggDecoder* aDecoder) :
mDecoder(aDecoder),
mPlayer(0),
mPlayStartTime(),
mPauseStartTime(),
mPauseDuration(0),
mPlaying(PR_FALSE),
mCallbackPeriod(1.0),
@@ -728,16 +794,17 @@ nsOggDecodeStateMachine::nsOggDecodeStat
mDuration(-1),
mSeekable(PR_TRUE),
mPositionChangeQueued(PR_FALSE),
mDecodingCompleted(PR_FALSE),
mExitStepDecodeThread(PR_FALSE),
mBufferExhausted(PR_FALSE),
mGotDurationFromHeader(PR_FALSE)
{
+ PreloadTables();
}
nsOggDecodeStateMachine::~nsOggDecodeStateMachine()
{
while (!mDecodedFrames.IsEmpty()) {
delete mDecodedFrames.Pop();
}
oggplay_close(mPlayer);
@@ -1015,22 +1082,102 @@ void nsOggDecodeStateMachine::PlayVideo(
mDecoder->SetRGBData(aFrame->mVideoWidth, aFrame->mVideoHeight,
mFramerate, mAspectRatio, buffer.forget());
// Don't play the frame's video data more than once.
aFrame->ClearVideoHeader();
}
}
+void nsOggDecodeStateMachine::CalculateFFT(nsTArray<float>& aData, nsTArray<float>& aSpectrum)
+{
+ nsTArray<float> signal;
+ nsTArray<float> real;
+ nsTArray<float> imag;
+
+ // Assuming interlaced stereo channels, need to split and merge into a stero-mix mono signal
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++)
+ signal.AppendElement((aData[2*i] + aData[2*i+1]) / 2);
+
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE; i++) {
+ real.AppendElement(signal[mReverseTable[i]]);
+ imag.AppendElement(0);
+ }
+
+ PRUint32 halfSize = 1;
+ float phaseShiftStepReal;
+ float phaseShiftStepImag;
+ float currentPhaseShiftReal;
+ float currentPhaseShiftImag;
+ PRUint32 i;
+ PRUint32 off;
+ float tr;
+ float ti;
+ float tmpReal;
+
+ while (halfSize < FFT_TABLE_SIZE) {
+ phaseShiftStepReal = mCosTable[halfSize];
+ phaseShiftStepImag = mSinTable[halfSize];
+ currentPhaseShiftReal = 1.0;
+ currentPhaseShiftImag = 0.0;
+
+ for (PRUint32 fftStep = 0; fftStep < halfSize; fftStep++) {
+ i = fftStep;
+
+ while (i < FFT_TABLE_SIZE) {
+ off = i + halfSize;
+ tr = (currentPhaseShiftReal * real[off]) - (currentPhaseShiftImag * imag[off]);
+ ti = (currentPhaseShiftReal * imag[off]) + (currentPhaseShiftImag * real[off]);
+
+ real[off] = real[i] - tr;
+ imag[off] = imag[i] - ti;
+ real[i] += tr;
+ imag[i] += ti;
+
+ i += halfSize << 1;
+ }
+
+ tmpReal = currentPhaseShiftReal;
+ currentPhaseShiftReal = (tmpReal * phaseShiftStepReal) - (currentPhaseShiftImag * phaseShiftStepImag);
+ currentPhaseShiftImag = (tmpReal * phaseShiftStepImag) + (currentPhaseShiftImag * phaseShiftStepReal);
+ }
+
+ halfSize = halfSize << 1;
+ }
+
+ // Calculate hypotenuse of the complex number with pythagorean theorem
+ for (PRUint32 i = 0; i < FFT_TABLE_SIZE / 2; i++) {
+ aSpectrum.AppendElement(sqrt(real[i] * real[i] + imag[i] * imag[i]) / FFT_TABLE_SIZE);
+ }
+}
+
void nsOggDecodeStateMachine::PlayAudio(FrameData* aFrame)
{
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
if (!mAudioStream)
return;
+ if (aFrame->mAudioData.Length() > 0) {
+ // Inform the element that we've written sound data.
+ nsTArray<float> spectrum;
+ nsTArray<float> frameBuffer = aFrame->mAudioData;
+
+ // Pad all frames to 4096 if they are less (some will be larger, that's OK).
+ if (frameBuffer.Length() < 4096) {
+ PRUint32 d = 4096 - frameBuffer.Length();
+ for (PRUint32 i = 0; i < d; i++)
+ frameBuffer.AppendElement(0);
+ }
+
+ CalculateFFT(frameBuffer, spectrum);
+
+ nsCOMPtr<nsIRunnable> event =
+ new nsAudioWrittenEventRunner(mDecoder, frameBuffer, spectrum);
+ NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+ }
aFrame->Write(mAudioStream);
}
void nsOggDecodeStateMachine::OpenAudioStream()
{
PR_ASSERT_CURRENT_THREAD_IN_MONITOR(mDecoder->GetMonitor());
mAudioStream = new nsAudioStream();
if (!mAudioStream) {
@@ -1902,29 +2049,47 @@ void nsOggDecodeStateMachine::SetTracksA
}
if (mAudioTrack != -1 &&
oggplay_set_track_active(mPlayer, mAudioTrack) < 0) {
LOG(PR_LOG_ERROR, ("%p Could not set track %d active", mDecoder, mAudioTrack));
}
}
+void nsOggDecodeStateMachine::Write(PRUint32 count, float *valueArray)
+{
+ if (!mAudioStream) {
+ mAudioStream = new nsAudioStream();
+// mAudioStream->Init(mAudioChannels, mAudioRate, nsAudioStream::FORMAT_FLOAT32);
+ mAudioStream->Init(1, 44100, nsAudioStream::FORMAT_FLOAT32);
+// mAudioStream->SetVolume(mVolume);
+ mAudioStream->SetVolume(1);
+ }
+ mAudioStream->Write(valueArray, count);
+}
+
NS_IMPL_THREADSAFE_ISUPPORTS1(nsOggDecoder, nsIObserver)
void nsOggDecoder::Pause()
{
nsAutoMonitor mon(mMonitor);
if (mPlayState == PLAY_STATE_SEEKING || mPlayState == PLAY_STATE_ENDED) {
mNextState = PLAY_STATE_PAUSED;
return;
}
ChangeState(PLAY_STATE_PAUSED);
}
+void nsOggDecoder::Write(PRUint32 count, float *valueArray)
+{
+ if (mDecodeStateMachine)
+ mDecodeStateMachine->Write(count, valueArray);
+}
+
void nsOggDecoder::SetVolume(float volume)
{
nsAutoMonitor mon(mMonitor);
mInitialVolume = volume;
if (mDecodeStateMachine) {
mDecodeStateMachine->SetVolume(volume);
}
@@ -2135,16 +2300,24 @@ nsMediaStream* nsOggDecoder::GetCurrentS
already_AddRefed<nsIPrincipal> nsOggDecoder::GetCurrentPrincipal()
{
if (!mReader)
return nsnull;
return mReader->Stream()->GetCurrentPrincipal();
}
+ void nsOggDecoder::AudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum)
+{
+ if (mShuttingDown)
+ return;
+
+ mElement->NotifyAudioWritten(aFrameBuffer, aSpectrum);
+}
+
void nsOggDecoder::MetadataLoaded()
{
if (mShuttingDown)
return;
// Only inform the element of MetadataLoaded if not doing a load() in order
// to fulfill a seek, otherwise we'll get multiple metadataloaded events.
PRBool notifyElement = PR_TRUE;
diff --git a/content/media/ogg/nsOggDecoder.h b/content/media/ogg/nsOggDecoder.h
--- a/content/media/ogg/nsOggDecoder.h
+++ b/content/media/ogg/nsOggDecoder.h
@@ -325,16 +325,18 @@ class nsOggDecoder : public nsMediaDecod
virtual nsresult Seek(float time);
virtual nsresult PlaybackRateChanged();
virtual void Pause();
virtual void SetVolume(float volume);
virtual float GetDuration();
+ virtual void Write(PRUint32 count, float *valueArray);
+
virtual nsMediaStream* GetCurrentStream();
virtual already_AddRefed<nsIPrincipal> GetCurrentPrincipal();
virtual void NotifySuspendedStatusChanged();
virtual void NotifyBytesDownloaded();
virtual void NotifyDownloadEnded(nsresult aStatus);
// Called by nsChannelReader on the decoder thread
void NotifyBytesConsumed(PRInt64 aBytes);
@@ -383,16 +385,18 @@ class nsOggDecoder : public nsMediaDecod
// Tells our nsMediaStream to put all loads in the background.
virtual void MoveLoadsToBackground();
// Stop the state machine thread and drop references to the thread,
// state machine and channel reader.
void Stop();
+ void AudioWritten(nsTArray<float>& aFrameBuffer, nsTArray<float>& aSpectrum);
+
protected:
// Returns the monitor for other threads to synchronise access to
// state.
PRMonitor* GetMonitor()
{
return mMonitor;
}
diff --git a/content/media/wave/nsWaveDecoder.cpp b/content/media/wave/nsWaveDecoder.cpp
--- a/content/media/wave/nsWaveDecoder.cpp
+++ b/content/media/wave/nsWaveDecoder.cpp
@@ -1251,16 +1251,22 @@ void
nsWaveDecoder::Pause()
{
if (mPlaybackStateMachine) {
mPlaybackStateMachine->Pause();
}
}
void
+nsWaveDecoder::Write(PRUint32 count, float *valueArray)
+{
+ return;
+}
+
+void
nsWaveDecoder::SetVolume(float aVolume)
{
mInitialVolume = aVolume;
if (mPlaybackStateMachine) {
mPlaybackStateMachine->SetVolume(aVolume);
}
}
diff --git a/content/media/wave/nsWaveDecoder.h b/content/media/wave/nsWaveDecoder.h
--- a/content/media/wave/nsWaveDecoder.h
+++ b/content/media/wave/nsWaveDecoder.h
@@ -160,16 +160,18 @@ class nsWaveDecoder : public nsMediaDeco
virtual float GetDuration();
// Set the audio playback volume; must be in range [0.0, 1.0].
virtual void SetVolume(float aVolume);
virtual nsresult Play();
virtual void Pause();
+ virtual void Write(PRUint32 count, float *valueArray);
+
// Set the current time of the media to aTime. This may cause mStream to
// create a new channel to fetch data from the appropriate position in the
// stream.
virtual nsresult Seek(float aTime);
// Report whether the decoder is currently seeking.
virtual PRBool IsSeeking() const;
diff --git a/dom/base/nsDOMClassInfo.cpp b/dom/base/nsDOMClassInfo.cpp
--- a/dom/base/nsDOMClassInfo.cpp
+++ b/dom/base/nsDOMClassInfo.cpp
@@ -228,16 +228,17 @@
#include "nsIDOMBeforeUnloadEvent.h"
#include "nsIDOMMutationEvent.h"
#include "nsIDOMSmartCardEvent.h"
#include "nsIDOMXULCommandEvent.h"
#include "nsIDOMPageTransitionEvent.h"
#include "nsIDOMMessageEvent.h"
#include "nsPaintRequest.h"
#include "nsIDOMNotifyPaintEvent.h"
+#include "nsIDOMNotifyAudioWrittenEvent.h"
#include "nsIDOMScrollAreaEvent.h"
#include "nsIDOMNSDocumentStyle.h"
#include "nsIDOMDocumentRange.h"
#include "nsIDOMDocumentTraversal.h"
#include "nsIDOMDocumentXBL.h"
#include "nsIDOMDocumentView.h"
#include "nsIDOMElementCSSInlineStyle.h"
#include "nsIDOMLinkStyle.h"
@@ -1303,16 +1304,22 @@ static nsDOMClassInfoData sClassInfoData
// data transfer for drag and drop
NS_DEFINE_CLASSINFO_DATA(DataTransfer, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
NS_DEFINE_CLASSINFO_DATA(NotifyPaintEvent, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
+ NS_DEFINE_CLASSINFO_DATA(NotifyAudioWrittenEvent, nsDOMGenericSH,
+ DOM_DEFAULT_SCRIPTABLE_FLAGS)
+
+ NS_DEFINE_CLASSINFO_DATA(AudioData, nsDOMGenericSH,
+ DOM_DEFAULT_SCRIPTABLE_FLAGS)
+
NS_DEFINE_CLASSINFO_DATA(SimpleGestureEvent, nsDOMGenericSH,
DOM_DEFAULT_SCRIPTABLE_FLAGS)
#ifdef MOZ_MATHML
NS_DEFINE_CLASSINFO_DATA_WITH_NAME(MathMLElement, Element, nsElementSH,
ELEMENT_SCRIPTABLE_FLAGS)
#endif
@@ -3682,16 +3689,25 @@ nsDOMClassInfo::Init()
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNSDataTransfer)
DOM_CLASSINFO_MAP_END
DOM_CLASSINFO_MAP_BEGIN(NotifyPaintEvent, nsIDOMNotifyPaintEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNotifyPaintEvent)
DOM_CLASSINFO_EVENT_MAP_ENTRIES
DOM_CLASSINFO_MAP_END
+ DOM_CLASSINFO_MAP_BEGIN(NotifyAudioWrittenEvent, nsIDOMNotifyAudioWrittenEvent)
+ DOM_CLASSINFO_MAP_ENTRY(nsIDOMNotifyAudioWrittenEvent)
+ DOM_CLASSINFO_EVENT_MAP_ENTRIES
+ DOM_CLASSINFO_MAP_END
+
+ DOM_CLASSINFO_MAP_BEGIN(AudioData, nsIDOMAudioData)
+ DOM_CLASSINFO_MAP_ENTRY(nsIDOMAudioData)
+ DOM_CLASSINFO_MAP_END
+
DOM_CLASSINFO_MAP_BEGIN(SimpleGestureEvent, nsIDOMSimpleGestureEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMSimpleGestureEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMMouseEvent)
DOM_CLASSINFO_MAP_ENTRY(nsIDOMNSMouseEvent)
DOM_CLASSINFO_UI_EVENT_MAP_ENTRIES
DOM_CLASSINFO_MAP_END
#ifdef MOZ_MATHML
diff --git a/dom/base/nsDOMClassInfoID.h b/dom/base/nsDOMClassInfoID.h
--- a/dom/base/nsDOMClassInfoID.h
+++ b/dom/base/nsDOMClassInfoID.h
@@ -450,16 +450,20 @@ enum nsDOMClassInfoID {
// DOM Traversal NodeIterator class
eDOMClassInfo_NodeIterator_id,
eDOMClassInfo_DataTransfer_id,
eDOMClassInfo_NotifyPaintEvent_id,
+ eDOMClassInfo_NotifyAudioWrittenEvent_id,
+
+ eDOMClassInfo_AudioData_id,
+
eDOMClassInfo_SimpleGestureEvent_id,
#ifdef MOZ_MATHML
eDOMClassInfo_MathMLElement_id,
#endif
eDOMClassInfo_Worker_id,
diff --git a/dom/interfaces/events/Makefile.in b/dom/interfaces/events/Makefile.in
--- a/dom/interfaces/events/Makefile.in
+++ b/dom/interfaces/events/Makefile.in
@@ -71,16 +71,17 @@ XPIDLSRCS = \
nsIDOMPopupBlockedEvent.idl \
nsIDOMBeforeUnloadEvent.idl \
nsIDOMNSEventTarget.idl \
nsIDOMSmartCardEvent.idl \
nsIDOMPageTransitionEvent.idl \
nsIDOMCommandEvent.idl \
nsIDOMMessageEvent.idl \
nsIDOMNotifyPaintEvent.idl \
+ nsIDOMNotifyAudioWrittenEvent.idl \
nsIDOMPaintRequest.idl \
nsIDOMPaintRequestList.idl \
nsIDOMSimpleGestureEvent.idl \
nsIDOMNSMouseEvent.idl \
nsIDOMOrientationEvent.idl \
nsIDOMScrollAreaEvent.idl \
$(NULL)
diff --git a/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl b/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl
new file mode 100644
--- /dev/null
+++ b/dom/interfaces/events/nsIDOMNotifyAudioWrittenEvent.idl
@@ -0,0 +1,23 @@
+#include "nsIDOMEvent.idl"
+#include "domstubs.idl"
+
+[scriptable, uuid(9d4872bc-1aba-46c6-ad52-00ea927e08fb)]
+interface nsIDOMAudioData : nsISupports
+{
+ readonly attribute unsigned long length;
+ float item(in unsigned long index);
+};
+
+
+[scriptable, uuid(6250652d-7a6a-49a4-a2ee-9114e1e83427)]
+interface nsIDOMNotifyAudioWrittenEvent : nsIDOMEvent
+{
+ readonly attribute nsIDOMAudioData mozFrameBuffer;
+ readonly attribute nsIDOMAudioData mozSpectrum;
+
+ void initAudioWrittenEvent(in DOMString typeArg,
+ in boolean canBubbleArg,
+ in boolean cancelableArg,
+ in nsIDOMAudioData frameBuffer,
+ in nsIDOMAudioData spectrum);
+};
diff --git a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
--- a/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
+++ b/dom/interfaces/html/nsIDOMHTMLMediaElement.idl
@@ -101,9 +101,15 @@ interface nsIDOMHTMLMediaElement : nsIDO
// Mozilla extension: load data from another media element. This is like
// load() but we don't run the resource selection algorithm; instead
// we just set our source to other's currentSrc. This is optimized
// so that this element will get access to all of other's cached/
// buffered data. In fact any future data downloaded by this element or
// other will be sharable by both elements.
void mozLoadFrom(in nsIDOMHTMLMediaElement other);
+
+ // Allow content to push audio data to the stream directly. Must call mozSetup first.
+ void mozWriteAudio(in PRUint32 count, [array, size_is(count)] in float valueArray);
+
+ // Setup the audio stream for writing
+ void mozSetup(in PRUint32 channels, in PRUint32 rate, in float volume);
};
diff --git a/widget/public/nsGUIEvent.h b/widget/public/nsGUIEvent.h
--- a/widget/public/nsGUIEvent.h
+++ b/widget/public/nsGUIEvent.h
@@ -395,18 +395,19 @@ class nsHashKey;
#define NS_CANPLAYTHROUGH (NS_MEDIA_EVENT_START+12)
#define NS_SEEKING (NS_MEDIA_EVENT_START+13)
#define NS_SEEKED (NS_MEDIA_EVENT_START+14)
#define NS_TIMEUPDATE (NS_MEDIA_EVENT_START+15)
#define NS_ENDED (NS_MEDIA_EVENT_START+16)
#define NS_RATECHANGE (NS_MEDIA_EVENT_START+17)
#define NS_DURATIONCHANGE (NS_MEDIA_EVENT_START+18)
#define NS_VOLUMECHANGE (NS_MEDIA_EVENT_START+19)
-#define NS_MEDIA_ABORT (NS_MEDIA_EVENT_START+20)
-#define NS_MEDIA_ERROR (NS_MEDIA_EVENT_START+21)
+#define NS_AUDIOWRITTEN (NS_MEDIA_EVENT_START+20)
+#define NS_MEDIA_ABORT (NS_MEDIA_EVENT_START+21)
+#define NS_MEDIA_ERROR (NS_MEDIA_EVENT_START+22)
#endif // MOZ_MEDIA
// paint notification events
#define NS_NOTIFYPAINT_START 3400
#define NS_AFTERPAINT (NS_NOTIFYPAINT_START)
// Simple gesture events
#define NS_SIMPLE_GESTURE_EVENT_START 3500
