Skip to content

Commit

Permalink
+ batch_size
Browse files Browse the repository at this point in the history
  • Loading branch information
@kanasimi
kanasimi committed Feb 4, 2024
1 parent 2aec528 commit 633b66f
Show file tree
Hide file tree
Showing 5 changed files with 263 additions and 78 deletions.
108 changes: 76 additions & 32 deletions Wikiapi.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1129,7 +1129,7 @@ assert('Mexico' in page_list[Wikiapi.KEY_subcategories], 'list category tree: [[
* @example <caption>Get all sub-categories of [[Category:Echinodermata]] with depth=2.</caption>
// <code>
const wiki = new Wikiapi('commons');
const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flated_subcategories: true })).flated_subcategories;
const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flat_subcategories: true })).flat_subcategories;
// </code>
*
* @memberof Wikiapi.prototype
Expand Down Expand Up @@ -1728,13 +1728,38 @@ function Wikiapi_run_SQL(SQL, for_each_row/* , options */) {

function Wikiapi_setup_layout_elements(options) {
function Wikiapi_setup_layout_elements_executor(resolve, reject) {
// const wiki = this[KEY_wiki_session];
wiki_API.setup_layout_elements(resolve, this.append_session_to_options(options));
const wiki = this[KEY_wiki_session];
wiki.setup_layout_elements(resolve, this.append_session_to_options(options));
}

return new Promise(Wikiapi_setup_layout_elements_executor.bind(this));
}

/**<code>
layout_element = CeL.wiki.parse('{{Authority control}}');
await wiki_session.setup_layout_element_to_insert(layout_element);
page_data = await wiki.page(page_data);
parsed = page_data.parse();
parsed.insert_layout_element(layout_element);
parsed.toString();
</code>*/
function Wikiapi_setup_layout_element_to_insert(layout_element, options) {
function Wikiapi_setup_layout_element_to_insert_executor(resolve, reject) {
const wiki = this[KEY_wiki_session];
wiki.setup_layout_element_to_insert(layout_element, (location, error) => {
if (error)
reject(error);
else
resolve(location);
}, options);
}

return new Promise(Wikiapi_setup_layout_element_to_insert_executor.bind(this));
}

// --------------------------------------------------------

/**
Expand Down Expand Up @@ -1956,6 +1981,7 @@ wiki.listen(function for_each_row() {
run_SQL: Wikiapi_run_SQL,

setup_layout_elements: Wikiapi_setup_layout_elements,
setup_layout_element_to_insert: Wikiapi_setup_layout_element_to_insert,
});

// wrapper for properties
Expand Down Expand Up @@ -1987,8 +2013,18 @@ console.log('All done.');
// <code>
let count = 0;
for await (const page_data of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC') })) {
if (++count > 5) break;
console.trace('page_data:', page_data);
if (++count > 5) break;
}
console.log('Done.');
// </code>
*
* @example <caption>Process all pages.</caption>
// <code>
let count = 0;
for await (const page_list of wiki.allpages({ namespace: 'Talk', apfrom: wiki.remove_namespace('ABC'), batch_size: 5 })) {
console.trace('page_list:', page_list);
if (++count > 2) break;
}
console.log('Done.');
// </code>
Expand Down Expand Up @@ -2022,6 +2058,27 @@ for (const type of wiki_API.list.type_list) {
//console.trace(arguments);

let done, page_queue = [], resolve_queue, waiting_promise, waiting_resolve;
// 餵頁面資料給 resolve()
function feed_page_data() {
if (resolve_queue.length === 0)
return;

if (!done && !(page_queue.length >= (options.batch_size > 0 ? options.batch_size : 1))) {
return;
}

let value;
// 由最早的開始給。
if (options.batch_size > 0) {
value = page_queue.splice(0, options.batch_size);
} else {
value = page_queue.shift();
}

// .shift(): 由最早的開始餵。
resolve_queue.shift()({ value });
}

function for_each_page(page_data) {
if (page_queue.abort)
return CeL.wiki.list.exit;
Expand All @@ -2033,21 +2090,15 @@ for (const type of wiki_API.list.type_list) {
if (!resolve_queue)
return;

if (resolve_queue.length === 0) {
page_queue.push(page_data);
//console.log(page_queue.length, 'pages in queue');
} else {
if (page_queue.length > 0) {
page_queue.push(page_data);
// 由最早的開始給。
page_data = page_queue.shift();
}
resolve_queue.shift()({ value: page_data });
}
page_queue.push(page_data);
//console.log(page_queue.length, 'pages in queue');
feed_page_data();

if (page_queue.length > 100) {
// 清掉前面累積的。
if (waiting_resolve)
waiting_resolve();
// 已經累積太多頁面資料，該緩緩了。
return new Promise(resolve => { waiting_resolve = resolve; });
}
}
Expand Down Expand Up @@ -2085,21 +2136,14 @@ for (const type of wiki_API.list.type_list) {
return { done };
return new Promise( /* executor */ function (resolve, reject) {
//console.trace(done, page_queue, resolve_queue);
if (page_queue.length === 0) {
resolve_queue.push(resolve);
if (waiting_resolve) {
waiting_resolve();
waiting_resolve = null;
}

} else {
if (resolve_queue.length > 0) {
// 依照標準實作不會到這裡來。
resolve_queue.push(resolve);
// 由最早的開始餵。
resolve = resolve_queue.shift();
}
resolve({ value: page_queue.shift() });
// 依照標準實作， resolve_queue.length === 0。
resolve_queue.push(resolve);
feed_page_data();

if (waiting_resolve) {
// 可以繼續接收頁面資料了。
waiting_resolve();
waiting_resolve = null;
}
});
},
Expand All @@ -2109,12 +2153,12 @@ for (const type of wiki_API.list.type_list) {
return { done };
},
throw(error) {
CeL.error('Wikiapi TODO: yet tested');
CeL.error('Wikiapi list TODO: Not yet tested');
page_queue.abort = done = true;
return { done };
}
};
}
};

return promise;
};
Expand Down
16 changes: 8 additions & 8 deletions docs/Wikiapi.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

<nav >

<h2><a href="index.html">Home</a></h2><h3>Classes</h3><ul><li><a href="Wikiapi.html">Wikiapi</a><ul class='methods'><li data-type='method'><a href="Wikiapi.html#category_tree">category_tree</a></li><li data-type='method'><a href="Wikiapi.html#convert_Chinese">convert_Chinese</a></li><li data-type='method'><a href="Wikiapi.html#data">data</a></li><li data-type='method'><a href="Wikiapi.html#delete">delete</a></li><li data-type='method'><a href="Wikiapi.html#download">download</a></li><li data-type='method'><a href="Wikiapi.html#edit">edit</a></li><li data-type='method'><a href="Wikiapi.html#edit_page">edit_page</a></li><li data-type='method'><a href="Wikiapi.html#for_each_page">for_each_page</a></li><li data-type='method'><a href="Wikiapi.html#get_featured_content">get_featured_content</a></li><li data-type='method'><a href="Wikiapi.html#listen">listen</a></li><li data-type='method'><a href="Wikiapi.html#login">login</a></li><li data-type='method'><a href="Wikiapi.html#move_page">move_page</a></li><li data-type='method'><a href="Wikiapi.html#move_to">move_to</a></li><li data-type='method'><a href="Wikiapi.html#new_data_entity">new_data_entity</a></li><li data-type='method'><a href="Wikiapi.html#page">page</a></li><li data-type='method'><a href="Wikiapi.html#purge">purge</a></li><li data-type='method'><a href="Wikiapi.html#query">query</a></li><li data-type='method'><a href="Wikiapi.html#redirects_here">redirects_here</a></li><li data-type='method'><a href="Wikiapi.html#redirects_root">redirects_root</a></li><li data-type='method'><a href="Wikiapi.html#register_redirects">register_redirects</a></li><li data-type='method'><a href="Wikiapi.html#search">search</a></li><li data-type='method'><a href="Wikiapi.html#site_name">site_name</a></li><li data-type='method'><a href="Wikiapi.html#SPARQL">SPARQL</a></li><li data-type='method'><a href="Wikiapi.html#tracking_revisions">tracking_revisions</a></li><li data-type='method'><a href="Wikiapi.html#upload">upload</a></li></ul></li></ul><h3>Global</h3><ul><li><a href="global.html#CeL">CeL</a></li><li><a href="global.html#KEY_SESSION">KEY_SESSION</a></li><li><a href="global.html#KEY_wiki_session">KEY_wiki_session</a></li><li><a href="global.html#modify_data_entity">modify_data_entity</a></li><li><a href="global.html#page_data_attributes">page_data_attributes</a></li><li><a href="global.html#reject_edit_error">reject_edit_error</a></li><li><a href="global.html#set_page_data_attributes">set_page_data_attributes</a></li><li><a href="global.html#setup_data_entity">setup_data_entity</a></li><li><a href="global.html#setup_wiki_session">setup_wiki_session</a></li><li><a href="global.html#wiki_API">wiki_API</a></li><li><a href="global.html#Wikiapi_for_each_page_in_list">Wikiapi_for_each_page_in_list</a></li><li><a href="global.html#Wikiapi_list">Wikiapi_list</a></li></ul>
<h2><a href="index.html">Home</a></h2><h3>Classes</h3><ul><li><a href="Wikiapi.html">Wikiapi</a><ul class='methods'><li data-type='method'><a href="Wikiapi.html#category_tree">category_tree</a></li><li data-type='method'><a href="Wikiapi.html#convert_Chinese">convert_Chinese</a></li><li data-type='method'><a href="Wikiapi.html#data">data</a></li><li data-type='method'><a href="Wikiapi.html#delete">delete</a></li><li data-type='method'><a href="Wikiapi.html#download">download</a></li><li data-type='method'><a href="Wikiapi.html#edit">edit</a></li><li data-type='method'><a href="Wikiapi.html#edit_page">edit_page</a></li><li data-type='method'><a href="Wikiapi.html#for_each_page">for_each_page</a></li><li data-type='method'><a href="Wikiapi.html#get_featured_content">get_featured_content</a></li><li data-type='method'><a href="Wikiapi.html#listen">listen</a></li><li data-type='method'><a href="Wikiapi.html#login">login</a></li><li data-type='method'><a href="Wikiapi.html#move_page">move_page</a></li><li data-type='method'><a href="Wikiapi.html#move_to">move_to</a></li><li data-type='method'><a href="Wikiapi.html#new_data_entity">new_data_entity</a></li><li data-type='method'><a href="Wikiapi.html#page">page</a></li><li data-type='method'><a href="Wikiapi.html#purge">purge</a></li><li data-type='method'><a href="Wikiapi.html#query">query</a></li><li data-type='method'><a href="Wikiapi.html#redirects_here">redirects_here</a></li><li data-type='method'><a href="Wikiapi.html#redirects_root">redirects_root</a></li><li data-type='method'><a href="Wikiapi.html#register_redirects">register_redirects</a></li><li data-type='method'><a href="Wikiapi.html#search">search</a></li><li data-type='method'><a href="Wikiapi.html#site_name">site_name</a></li><li data-type='method'><a href="Wikiapi.html#SPARQL">SPARQL</a></li><li data-type='method'><a href="Wikiapi.html#tracking_revisions">tracking_revisions</a></li><li data-type='method'><a href="Wikiapi.html#upload">upload</a></li></ul></li></ul><h3>Global</h3><ul><li><a href="global.html#CeL">CeL</a></li><li><a href="global.html#KEY_SESSION">KEY_SESSION</a></li><li><a href="global.html#KEY_wiki_session">KEY_wiki_session</a></li><li><a href="global.html#modify_data_entity">modify_data_entity</a></li><li><a href="global.html#page_data_attributes">page_data_attributes</a></li><li><a href="global.html#reject_edit_error">reject_edit_error</a></li><li><a href="global.html#set_page_data_attributes">set_page_data_attributes</a></li><li><a href="global.html#setup_data_entity">setup_data_entity</a></li><li><a href="global.html#setup_wiki_session">setup_wiki_session</a></li><li><a href="global.html#wiki_API">wiki_API</a></li><li><a href="global.html#Wikiapi_for_each_page_in_list">Wikiapi_for_each_page_in_list</a></li><li><a href="global.html#Wikiapi_list">Wikiapi_list</a></li><li><a href="global.html#Wikiapi_setup_layout_element_to_insert">Wikiapi_setup_layout_element_to_insert</a></li></ul>
</nav>

<div id="main">
Expand Down Expand Up @@ -446,7 +446,7 @@ <h5>Examples</h5>

<pre class="prettyprint"><code>// &lt;code>
const wiki = new Wikiapi('commons');
const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flated_subcategories: true })).flated_subcategories;
const all_sub_categories = (await wiki.category_tree('Echinodermata', { depth: 2, cmtype: 'subcat', get_flat_subcategories: true })).flat_subcategories;
// &lt;/code></code></pre>


Expand Down Expand Up @@ -1084,7 +1084,7 @@ <h4 class="name" id="delete"><span class="type-signature"></span>delete<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1844">line 1844</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1869">line 1869</a>
</li></ul></dd>


Expand Down Expand Up @@ -1551,7 +1551,7 @@ <h4 class="name" id="edit"><span class="type-signature"></span>edit<span class="

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1892">line 1892</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1917">line 1917</a>
</li></ul></dd>


Expand Down Expand Up @@ -2318,7 +2318,7 @@ <h4 class="name" id="get_featured_content"><span class="type-signature"></span>g

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1763">line 1763</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1788">line 1788</a>
</li></ul></dd>


Expand Down Expand Up @@ -2506,7 +2506,7 @@ <h4 class="name" id="listen"><span class="type-signature"></span>listen<span cla

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1925">line 1925</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1950">line 1950</a>
</li></ul></dd>


Expand Down Expand Up @@ -5215,7 +5215,7 @@ <h4 class="name" id="site_name"><span class="type-signature"></span>site_name<sp

<dt class="tag-source">Source:</dt>
<dd class="tag-source"><ul class="dummy"><li>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1817">line 1817</a>
<a href="wikiapi.js.html">wikiapi.js</a>, <a href="wikiapi.js.html#line1842">line 1842</a>
</li></ul></dd>


Expand Down Expand Up @@ -6143,7 +6143,7 @@ <h5>Returns:</h5>
<br class="clear">

<footer>
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Thu Jan 18 2024 05:11:02 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
Documentation generated by <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.6.10</a> on Sun Feb 04 2024 17:44:17 GMT+0800 (台北標準時間) using the <a href="https://github.com/clenemt/docdash">docdash</a> theme.
</footer>

<script>prettyPrint();</script>
Expand Down
Loading

0 comments on commit 633b66f

Please sign in to comment.