Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Salvage #13772 #13935

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions res/controllers/engine-api.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -301,4 +301,35 @@ declare namespace engine {
* SoftStart with low factors would take a while until sound is audible. [default = 1.0]
*/
function softStart(deck: number, activate: boolean, factor?: number): void;

enum WellKnownCharsets {
US_ASCII,
Latin1,
ISO_8859_1,
Latin9,
ISO_8859_15,
UCS2, // with prepended Byte-Order-Mark
ISO_10646_UCS_2, // with prepended Byte-Order-Mark
UTF_8,
UTF_16BE,
UTF_16LE,
}

/**
* Converts a string into another charset.
*
* This function is useful to display text on a device that does not make use of UTF-8.
* Available charset names are listed here: http://www.iana.org/assignments/character-sets/character-sets.xhtml.
* Characters that are unsupported by target charset will be transformed to null character (0x00).
* @param targetCharset The charset to encode the string into.
* @param value The string to encode
* @returns The converted String as an array of bytes. Will return an empty buffer on conversion error or unavailable charset.
*/
function convertCharset(targetCharset: string, value: string): ArrayBuffer
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wait this is still documented even though it's now an internal API?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

forgot to remove. thanks for the reminder.


/**
* @param value The string to encode
* @returns The converted String as an array of bytes. Will return an empty buffer on conversion error or unavailable charset.
*/
function convertCharset(targetCharset: WellKnownCharsets, value: string): ArrayBuffer
}
Original file line number Diff line number Diff line change
Expand Up @@ -338,8 +338,10 @@ bool ControllerScriptEngineLegacy::initialize() {
ControllerScriptInterfaceLegacy* legacyScriptInterface =
new ControllerScriptInterfaceLegacy(this, m_logger);

engineGlobalObject.setProperty(
"engine", m_pJSEngine->newQObject(legacyScriptInterface));
auto engine = m_pJSEngine->newQObject(legacyScriptInterface);
auto meta = m_pJSEngine->newQMetaObject(&ControllerScriptInterfaceLegacy::staticMetaObject);
engine.setProperty("WellKnownCharsets", meta);
engineGlobalObject.setProperty("engine", m_pJSEngine->newQObject(legacyScriptInterface));

#ifdef MIXXX_USE_QML
if (m_bQmlMode) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
#include "controllerscriptinterfacelegacy.h"

#if QT_VERSION < QT_VERSION_CHECK(6, 4, 0)
#include <QTextCodec>
#else
#include <QStringEncoder>
#endif
#include <gsl/pointers>

#include "control/controlobject.h"
Expand Down Expand Up @@ -1052,3 +1057,55 @@ void ControllerScriptInterfaceLegacy::softStart(int deck, bool activate, double
// activate the ramping in scratchProcess()
m_ramp[deck] = true;
}

QByteArray ControllerScriptInterfaceLegacy::convertCharset(
const ControllerScriptInterfaceLegacy::WellKnownCharsets targetCharset,
const QString& value) {
switch (targetCharset) {
case WellKnownCharsets::US_ASCII:
return convertCharset(QStringLiteral("US-ASCII"), value);
case WellKnownCharsets::Latin1:
case WellKnownCharsets::ISO_8859_1:
return convertCharset(QStringLiteral("ISO-8859-1"), value);
case WellKnownCharsets::Latin9:
case WellKnownCharsets::ISO_8859_15:
return convertCharset(QStringLiteral("ISO-8859-15"), value);
case WellKnownCharsets::UCS2:
case WellKnownCharsets::ISO_10646_UCS_2:
return convertCharset(QStringLiteral("ISO-10646-UCS-2"), value);
case WellKnownCharsets::UTF_8:
return convertCharset(QStringLiteral("UTF-8"), value);
case WellKnownCharsets::UTF_16BE:
return convertCharset(QStringLiteral("UTF-16BE"), value);
case WellKnownCharsets::UTF_16LE:
return convertCharset(QStringLiteral("UTF-16LE"), value);
}
m_pScriptEngineLegacy->logOrThrowError(QStringLiteral("Unknown charset specified"));
return QByteArray();
}

QByteArray ControllerScriptInterfaceLegacy::convertCharset(
const QString& targetCharset, const QString& value) {
#if QT_VERSION < QT_VERSION_CHECK(6, 4, 0)
QByteArray encoderNameArray = targetCharset.toUtf8();
auto* pCodec = QTextCodec::codecForName(encoderNameArray);
if (!pCodec) {
qCWarning(m_logger) << "Unable to open encoder";
return QByteArray();
}
return std::unique_ptr(pCodec->makeEncoder())->fromUnicode(value);
#else
#if QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)
QAnyStringView encoderName = QAnyStringView(targetCharset);
#else
QByteArray encoderNameArray = targetCharset.toUtf8();
const char* encoderName = encoderNameArray.constData();
#endif
QStringEncoder fromUtf16 = QStringEncoder(encoderName);
if (!fromUtf16.isValid()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why you removed the flags here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

see the commit message, writing the replacement char is better than replacing with null bytes IMO.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This may be a problem. During the tests I noticed that replacement char varies between Ubuntu and Fedora (maybe between Qt versions). Replace invalid chars with \0x00 is the most predictable option.

Copy link
Member Author

@Swiftb0y Swiftb0y Nov 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are you sure? I think it depends on the encoding. The Qt docs say they use QChar::ReplacementCharacter or a question mark.

Copy link
Contributor

@christophehenry christophehenry Nov 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is what I noticed here. The relevant commit is probably this one. Although I tested your branch and the tests pass here too so I'm not sure anymore.

qCWarning(m_logger) << "Unable to open encoder";
return QByteArray();
}
return fromUtf16(value);
#endif
}
21 changes: 21 additions & 0 deletions src/controllers/scripting/legacy/controllerscriptinterfacelegacy.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,20 @@ class ConfigKey;
class ControllerScriptInterfaceLegacy : public QObject {
Q_OBJECT
public:
enum class WellKnownCharsets {
US_ASCII,
Latin1,
ISO_8859_1,
Latin9,
ISO_8859_15,
UCS2,
ISO_10646_UCS_2,
UTF_8,
UTF_16BE,
UTF_16LE,
};
Q_ENUM(WellKnownCharsets)

ControllerScriptInterfaceLegacy(ControllerScriptEngineLegacy* m_pEngine,
const RuntimeLoggingCategory& logger);

Expand Down Expand Up @@ -72,6 +86,13 @@ class ControllerScriptInterfaceLegacy : public QObject {
const double rate = -10.0);
Q_INVOKABLE void softStart(const int deck, bool activate, double factor = 1.0);

Q_INVOKABLE QByteArray convertCharset(
const ControllerScriptInterfaceLegacy::WellKnownCharsets
targetCharset,
const QString& value);

Q_INVOKABLE QByteArray convertCharset(const QString& targetCharset, const QString& value);
JoergAtGithub marked this conversation as resolved.
Show resolved Hide resolved

bool removeScriptConnection(const ScriptConnection& conn);
/// Execute a ScriptConnection's JS callback
void triggerScriptConnection(const ScriptConnection& conn);
Expand Down
81 changes: 81 additions & 0 deletions src/test/controllerscriptenginelegacy_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>

#include <QByteArrayView>
#include <QMetaEnum>
#include <QScopedPointer>
#include <QTemporaryFile>
#include <QThread>
Expand All @@ -12,6 +14,7 @@

#include "control/controlobject.h"
#include "control/controlpotmeter.h"
#include "controllers/scripting/legacy/controllerscriptinterfacelegacy.h"
#ifdef MIXXX_USE_QML
#include <QQuickItem>

Expand Down Expand Up @@ -658,6 +661,84 @@ TEST_F(ControllerScriptEngineLegacyTest, connectionExecutesWithCorrectThisObject
EXPECT_DOUBLE_EQ(1.0, pass->get());
}

TEST_F(ControllerScriptEngineLegacyTest, convertCharsetUndefinedOnUnknownCharset) {
const auto result = evaluate("engine.convertCharset('NULL', 'Hello!')");

EXPECT_EQ(qjsvalue_cast<QByteArray>(result), QByteArrayView(""));
}
Comment on lines +664 to +668
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah... Not quite sure what to do here honestly. It seems that there are implicit conversions to enums happening, which isn't great if it just silently succeeds. I wonder what value plain undefined results in? I'd guess undefined->0->US_ASCII?


TEST_F(ControllerScriptEngineLegacyTest, convertCharsetCorrectValueWellKnown) {
const auto result = evaluate(
"engine.convertCharset(engine.WellKnownCharsets.Latin9, 'Hello!')");

// ISO-8859-15 ecoded 'Hello!'
EXPECT_EQ(qjsvalue_cast<QByteArray>(result),
QByteArrayView::fromArray({'\x48', '\x65', '\x6c', '\x6c', '\x6f', '\x21'}));
}

TEST_F(ControllerScriptEngineLegacyTest, convertCharsetCorrectValueStringCharset) {
const auto result = evaluate("engine.convertCharset('ISO-8859-15', 'Hello!')");

// ISO-8859-15 ecoded 'Hello!'
EXPECT_EQ(qjsvalue_cast<QByteArray>(result),
QByteArrayView::fromArray({'\x48', '\x65', '\x6c', '\x6c', '\x6f', '\x21'}));
}

TEST_F(ControllerScriptEngineLegacyTest, convertCharsetUnsupportedChars) {
auto result = qjsvalue_cast<QByteArray>(
evaluate("engine.convertCharset('ISO-8859-15', 'مايأ نامز')"));

EXPECT_EQ(result,
QByteArrayView::fromArray(
{'\x00', '\x00', '\x00', '\x00', '\x20', '\x00', '\x00', '\x00', '\x00'}));
}

#define COMPLICATEDSTRINGLITERAL "Hello, 世界! שלום! こんにちは! 안녕하세요! 😊"

static int convertedCharsetForString(ControllerScriptInterfaceLegacy::WellKnownCharsets charset) {
// the expected length after conversion of COMPLICATEDSTRINGLITERAL
using enum ControllerScriptInterfaceLegacy::WellKnownCharsets;
switch (charset) {
case US_ASCII:
case Latin9:
case ISO_8859_15:
return 32;
case Latin1:
case ISO_8859_1:
return 33;
case UTF_8:
return 63;
case UTF_16BE:
case UTF_16LE:
return 66;
case UCS2:
case ISO_10646_UCS_2:
return 68;
}
// unreachable (TODO assert false?)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That assert would make sense IMO

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, still TODO

return 0;
}

TEST_F(ControllerScriptEngineLegacyTest, convertCharsetAllWellKnownCharsets) {
QMetaEnum charsetEnumEntry = QMetaEnum::fromType<
ControllerScriptInterfaceLegacy::WellKnownCharsets>();

for (int i = 0; i < charsetEnumEntry.keyCount(); ++i) {
QString key = charsetEnumEntry.key(i);
auto enumValue =
static_cast<ControllerScriptInterfaceLegacy::WellKnownCharsets>(
charsetEnumEntry.value(i));
QString source = QStringLiteral(
"engine.convertCharset(engine.WellKnownCharsets.%1, "
"'" COMPLICATEDSTRINGLITERAL "')")
.arg(key);
auto result = qjsvalue_cast<QByteArray>(evaluate(source));
EXPECT_EQ(result.size(), convertedCharsetForString(enumValue))
<< "Unexpected length of converted string for encoding: '"
<< key.toStdString() << "'";
}
}

#ifdef MIXXX_USE_QML
class MockScreenRender : public ControllerRenderingEngine {
public:
Expand Down
Loading