From edb0b77b78a173654fa1afebca546e207a6878a0 Mon Sep 17 00:00:00 2001 From: Martha Thompson <437455+MothOnMars@users.noreply.github.com> Date: Mon, 22 Mar 2021 16:43:45 -0400 Subject: [PATCH] SRCH-1915 fix Tweet.expire method (#671) --- app/models/tweet.rb | 11 ++-- ...234859_add_index_on_tweets_published_at.rb | 5 ++ db/structure.sql | 6 +- spec/models/tweet_spec.rb | 64 ++++++++++++------- 4 files changed, 57 insertions(+), 29 deletions(-) create mode 100644 db/migrate/20210317234859_add_index_on_tweets_published_at.rb diff --git a/app/models/tweet.rb b/app/models/tweet.rb index 8b403733f0..0e97566ce3 100644 --- a/app/models/tweet.rb +++ b/app/models/tweet.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + class Tweet < ApplicationRecord before_save :sanitize_tweet_text - belongs_to :twitter_profile, :primary_key => :twitter_id - validates_presence_of :tweet_id, :tweet_text, :published_at, :twitter_profile_id - validates_uniqueness_of :tweet_id + belongs_to :twitter_profile, primary_key: :twitter_id + validates :tweet_id, :tweet_text, :published_at, :twitter_profile_id, presence: true + validates :tweet_id, uniqueness: true serialize :urls, Array def sanitize_tweet_text @@ -30,6 +32,7 @@ def as_json(_options = {}) end def self.expire(days_back) - destroy_all(["published_at < ?", days_back.days.ago.beginning_of_day.to_s(:db)]) + where('published_at < ?', days_back.days.ago.beginning_of_day.to_s(:db)). + in_batches.destroy_all end end diff --git a/db/migrate/20210317234859_add_index_on_tweets_published_at.rb b/db/migrate/20210317234859_add_index_on_tweets_published_at.rb new file mode 100644 index 0000000000..9b4cc4836c --- /dev/null +++ b/db/migrate/20210317234859_add_index_on_tweets_published_at.rb @@ -0,0 +1,5 @@ +class AddIndexOnTweetsPublishedAt < ActiveRecord::Migration[5.2] + def change + add_index :tweets, :published_at + end +end diff --git a/db/structure.sql b/db/structure.sql index 9aeb51da74..a60e8d2017 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1041,7 +1041,8 @@ CREATE TABLE `tweets` ( `urls` mediumtext COLLATE utf8mb4_unicode_ci, PRIMARY KEY (`id`), UNIQUE KEY `index_tweets_on_tweet_id` (`tweet_id`), - KEY `index_tweets_on_twitter_profile_id` (`twitter_profile_id`) + KEY `index_tweets_on_twitter_profile_id` (`twitter_profile_id`), + KEY `index_tweets_on_published_at` (`published_at`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci ROW_FORMAT=DYNAMIC; /*!40101 SET character_set_client = @saved_cs_client */; DROP TABLE IF EXISTS `twitter_lists`; @@ -1929,6 +1930,7 @@ INSERT INTO `schema_migrations` (version) VALUES ('20191113214448'), ('20200121220041'), ('20200212183209'), -('20200728194854'); +('20200728194854'), +('20210317234859'); diff --git a/spec/models/tweet_spec.rb b/spec/models/tweet_spec.rb index fa2d1d80d2..541494d4bb 100644 --- a/spec/models/tweet_spec.rb +++ b/spec/models/tweet_spec.rb @@ -1,17 +1,15 @@ -require 'spec_helper' +# frozen_string_literal: true describe Tweet do - fixtures :affiliates - - before do - allow(Twitter).to receive(:user).and_return double('Twitter', id: 12345, name: 'USASearch', profile_image_url: 'http://some.gov/url') - @valid_attributes = { + let(:twitter_profile) { twitter_profiles(:usasearch) } + let(:valid_attributes) do + { tweet_id: 18700887835, - tweet_text: 'got a lovely surprise from @craftybeans. She sent me the best tshirt ever. http://www.flickr.com/photos/cindyli/4799054041/ ::giggles::', - published_at: Time.now + tweet_text: 'this is a tweet', + published_at: Time.now, + twitter_profile_id: twitter_profile.id } end - let(:profile) do TwitterProfile.create!(twitter_id: 12345, screen_name: 'USASearch', @@ -24,15 +22,19 @@ it { is_expected.to validate_presence_of :published_at } it { is_expected.to validate_presence_of :twitter_profile_id } - it 'should create new instance given valid attributes' do - tweet = described_class.create!(@valid_attributes.merge(twitter_profile_id: profile.id)) - expect(tweet.tweet_id).to eq(@valid_attributes[:tweet_id]) - expect(tweet.tweet_text).to eq(@valid_attributes[:tweet_text]) + describe 'schema' do + it { is_expected.to have_db_index(:published_at) } + end + + it 'creates new instance given valid attributes' do + tweet = described_class.create!(valid_attributes.merge(twitter_profile_id: profile.id)) + expect(tweet.tweet_id).to eq(valid_attributes[:tweet_id]) + expect(tweet.tweet_text).to eq(valid_attributes[:tweet_text]) is_expected.to validate_uniqueness_of :tweet_id end - it 'should sanitize tweet text' do + it 'sanitizes tweet text' do tweet = described_class.create!(tweet_text: "A tweet with \n http://t.co/h5vNlSdL and http://t.co/YQQSs9bb", tweet_id: 123456, published_at: Time.now, @@ -44,20 +46,20 @@ context 'when tweet can be traced back to at least one affiliate' do before do profile.affiliates << affiliates(:gobiernousa_affiliate) - @tweet = profile.tweets.create!(@valid_attributes) + @tweet = profile.tweets.create!(valid_attributes) end - it 'should use the locale for the first affiliate' do + it 'uses the locale for the first affiliate' do expect(@tweet.language).to eq('es') end end context 'when tweet cannot be traced back to at least one affiliate' do before do - @tweet = described_class.create!(@valid_attributes.merge(twitter_profile_id: profile.id)) + @tweet = described_class.create!(valid_attributes.merge(twitter_profile_id: profile.id)) end - it 'should use English' do + it 'uses English' do expect(@tweet.language).to eq('en') end end @@ -72,16 +74,32 @@ @tweet = described_class.create!(tweet_text: 'USA', tweet_id: 123456, published_at: Time.now, twitter_profile_id: 12345) end - it 'should output a properly formatted link to the tweet' do + it 'outputs a properly formatted link to the tweet' do expect(@tweet.url_to_tweet).to eq('https://twitter.com/USASearch/status/123456') end end describe '.expire(days_back)' do - it 'should destroy tweets that were published more than X days ago' do - expect(described_class).to receive(:destroy_all).with(['published_at < ?', 3.days.ago.beginning_of_day.to_s(:db)]) - described_class.expire(3) + subject(:expire) { described_class.expire(3) } + + context 'when tweets exist' do + before do + described_class.create!( + valid_attributes.merge( + tweet_text: 'old tweet', published_at: 4.days.ago, tweet_id: 1 + ) + ) + described_class.create!( + valid_attributes.merge( + tweet_text: 'new tweet', published_at: 1.minute.ago, tweet_id: 2 + ) + ) + end + + it 'destroys tweets that were published more than X days ago' do + expire + expect(described_class.pluck(:tweet_text)).to eq ['new tweet'] + end end end - end