Skip to content

Commit

Permalink
Fix issue #374 Error parsing some "Likes" posts
Browse files Browse the repository at this point in the history 
- There was an error that prevented the parsing of reblogged text posts with no added content.
- Now normal and reblogged text posts can be downloaded.
  • Loading branch information
@thomas694
thomas694 committed Nov 1, 2022
1 parent 5de83c0 commit a49353f
Show file tree
Hide file tree
Showing 4 changed files with 166 additions and 28 deletions.
4 changes: 2 additions & 2 deletions src/TumblThree/SharedAssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@

[assembly: ComVisible(false)]
[assembly: NeutralResourcesLanguage("en-US", UltimateResourceFallbackLocation.MainAssembly)]
[assembly: AssemblyVersion("2.8.3.0")]
[assembly: AssemblyFileVersion("2.8.3.0")]
[assembly: AssemblyVersion("2.9.0.0")]
[assembly: AssemblyFileVersion("2.9.0.0")]
7 changes: 7 additions & 0 deletions src/TumblThree/TumblThree.Applications/Crawler/AbstractTumblrCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,10 +434,17 @@ protected string RetrieveOriginalImageUrl(string url, int width, int height, boo
}
catch (WebException we)
{
errCnt++;
if (we.Status == WebExceptionStatus.RequestCanceled)
throw new NullReferenceException("RetrieveOriginalImageUrl request cancelled");
if (we.Response != null && ((HttpWebResponse)we.Response).StatusCode == HttpStatusCode.NotFound)
return url;
if (we.Response != null && (int)((HttpWebResponse)we.Response).StatusCode == 429)
{
throw new Exception("429 - Limit exceeded.");
}
lastError = we;
if (errCnt < 3) Thread.Sleep(errCnt * 10000);
}
catch (Exception e)
{
Expand Down
92 changes: 68 additions & 24 deletions src/TumblThree/TumblThree.Applications/Crawler/TumblrLikedByCrawler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,26 +331,27 @@ private async Task DownloadPage(List<DataModels.TumblrSearchJson.Data> posts)
try
{
Post data = null;
data = new Post()
{
Date = post.Date,
DateGmt = post.Date,
Type = "regular",
Id = post.Id,
Tags = post.Tags.ToList(),
Slug = post.Slug,
RegularTitle = post.Summary,
RebloggedFromName = "",
RebloggedRootName = "",
ReblogKey = post.ReblogKey,
UnixTimestamp = post.Timestamp,
Tumblelog = new TumbleLog2() { Name = post.BlogName },
UrlWithSlug = post.PostUrl
};
var countImagesVideos = CountImagesAndVideos(post.Content);
int index = -1;
foreach (var content in post.Content)
{
data = new Post()
{
Date = post.Date,
DateGmt = post.Date,
Type = ConvertContentTypeToPostType(content.Type),
Id = post.Id,
Tags = post.Tags.ToList(),
Slug = post.Slug,
RegularTitle = post.Summary,
RebloggedFromName = "",
RebloggedRootName = "",
ReblogKey = post.ReblogKey,
UnixTimestamp = post.Timestamp,
Tumblelog = new TumbleLog2() { Name = post.BlogName },
UrlWithSlug = post.PostUrl
};
data.Type = ConvertContentTypeToPostType(content.Type);
index += (countImagesVideos > 1) ? 1 : 0;
DownloadMedia(content, data, index);
AddInlinePhotoUrl(post, content, data);
Expand All @@ -373,23 +374,66 @@ private void DownloadText(DataModels.TumblrSearchJson.Data post, Post data)
if (Blog.DownloadText && new string[] { "regular", "quote", "note", "link", "conversation" }.Contains(post.OriginalType))
{
string text = "";
foreach (var content in post.Content)
if (post.Content.Count == 0)
{
foreach (var trail in post.Trail)
{
text += Environment.NewLine + trail.Blog.Name + "/" + trail.Post.Id + ":" + Environment.NewLine + Environment.NewLine;
foreach (var content in trail.Content)
{
if (content.Type == "text")
{
text += content.Text + Environment.NewLine + (content.SubType == "heading1" || content.SubType == "heading2" ? "" : Environment.NewLine);
}
}
}
}
else
{
if (content.Type == "text")
foreach (var content in post.Content)
{
text += content.Text + Environment.NewLine;
if (content.Type == "text")
{
text += content.Text + Environment.NewLine + (content.SubType == "heading1" || content.SubType == "heading2" ? "" : Environment.NewLine);
}
}
}
data.RegularBody = text;
data.RegularBody = text.Trim(Environment.NewLine.ToCharArray());
data.Type = post.OriginalType;

switch (post.OriginalType)
{
case "regular":
data.RegularTitle = post.Content?[0]?.Text;
data.RegularBody = string.Join(Environment.NewLine, post.Content.Skip(1).Select(s => s.Text));
text = tumblrJsonParser.ParseText(data);
AddToDownloadList(new TextPost(text, data.Id, data.UnixTimestamp.ToString()));
if (post.Content.Count == 0)
{
data.RegularTitle = $"{post.BlogName} reblogged {post.RebloggedFromName}/{post.RebloggedFromId}";
foreach (var trail in post.Trail)
{
text += Environment.NewLine + trail.Blog.Name + "/" + trail.Post.Id + ":" + Environment.NewLine + Environment.NewLine;
foreach (var content in trail.Content)
{
if (content.Type == "text")
{
text += content.Text + Environment.NewLine + (content.SubType == "heading1" || content.SubType == "heading2" ? "" : Environment.NewLine);
}
}
}
data.RegularBody = text.Trim(Environment.NewLine.ToCharArray());
}
else
{
data.RegularTitle = (post.Content?[0]?.SubType ?? "") == "heading1" ? post.Content?[0]?.Text : "";
data.RegularBody = string.Join("", post.Content
.Where(c => c.Type == "text")
.Skip((post.Content?[0]?.SubType ?? "") == "heading1" ? 1 : 0)
.Select(s => s.Text + Environment.NewLine + (s.SubType == "heading1" || s.SubType == "heading2" ? "" : Environment.NewLine)))
.Trim(Environment.NewLine.ToCharArray());
}
if (data.RegularTitle.Length != 0 || data.RegularBody.Length != 0)
{
text = tumblrJsonParser.ParseText(data);
AddToDownloadList(new TextPost(text, data.Id, data.UnixTimestamp.ToString()));
}
break;
case "quote":
data.QuoteText = post.Content?[0]?.Text;
Expand Down
91 changes: 89 additions & 2 deletions src/TumblThree/TumblThree.Applications/DataModels/TumblrSearch/TumblrSearchJson.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,11 +243,74 @@ public class Data
public IList<Layout> Layout { get; set; }

[JsonProperty("trail")]
public IList<object> Trail { get; set; }
public IList<Trail> Trail { get; set; }

[JsonProperty("placementId")]
public string PlacementId { get; set; }

[JsonProperty("rebloggedFromId")]
public string RebloggedFromId { get; set; }

[JsonProperty("rebloggedFromUrl")]
public string RebloggedFromUrl { get; set; }

[JsonProperty("rebloggedFromName")]
public string RebloggedFromName { get; set; }

[JsonProperty("rebloggedFromTitle")]
public string RebloggedFromTitle { get; set; }

[JsonProperty("rebloggedFromUuid")]
public string RebloggedFromUuid { get; set; }

[JsonProperty("rebloggedFromCanMessage")]
public bool RebloggedFromCanMessage { get; set; }

[JsonProperty("rebloggedFromShareLikes")]
public bool RebloggedFromShareLikes { get; set; }

[JsonProperty("rebloggedFromShareFollowing")]
public bool RebloggedFromShareFollowing { get; set; }

[JsonProperty("rebloggedFromCanBeFollowed")]
public bool RebloggedFromCanBeFollowed { get; set; }

[JsonProperty("rebloggedFromFollowing")]
public bool RebloggedFromFollowing { get; set; }

[JsonProperty("rebloggedRootId")]
public string RebloggedRootId { get; set; }

[JsonProperty("rebloggedRootUrl")]
public string RebloggedRootUrl { get; set; }

[JsonProperty("rebloggedRootName")]
public string RebloggedRootName { get; set; }

[JsonProperty("rebloggedRootTitle")]
public string RebloggedRootTitle { get; set; }

[JsonProperty("rebloggedRootUuid")]
public string RebloggedRootUuid { get; set; }

[JsonProperty("rebloggedRootCanMessage")]
public bool RebloggedRootCanMessage { get; set; }

[JsonProperty("rebloggedRootShareLikes")]
public bool RebloggedRootShareLikes { get; set; }

[JsonProperty("rebloggedRootShareFollowing")]
public bool RebloggedRootShareFollowing { get; set; }

[JsonProperty("rebloggedRootCanBeFollowed")]
public bool RebloggedRootCanBeFollowed { get; set; }

[JsonProperty("rebloggedRootFollowing")]
public bool RebloggedRootFollowing { get; set; }

[JsonProperty("likedTimestamp")]
public int LikedTimestamp { get; set; }

[JsonProperty("canEdit")]
public bool CanEdit { get; set; }

Expand Down Expand Up @@ -512,6 +575,9 @@ public class Content
[JsonProperty("type")]
public string Type { get; set; }

[JsonProperty("subtype")]
public string SubType { get; set; }

[JsonProperty("media")]
[JsonConverter(typeof(SingleOrArrayConverter<Medium>))]
public IList<Medium> Media { get; set; }
Expand Down Expand Up @@ -717,7 +783,7 @@ public class SourceAttribution
public object SyndicationId { get; set; }

[JsonProperty("deepLinks")]
public IList<object> DeepLinks { get; set; }
public object DeepLinks { get; set; }

[JsonProperty("appStoreIds")]
public AppStoreIds AppStoreIds { get; set; }
Expand Down Expand Up @@ -1540,4 +1606,25 @@ public class Links
[JsonProperty("next")]
public NextRequest Next { get; set; }
}

public class Trail
{
[JsonProperty("content")]
public IList<Content> Content { get; set; }

[JsonProperty("layout")]
public IList<Layout> Layout { get; set; }

[JsonProperty("post")]
public TrailPost Post { get; set; }

[JsonProperty("blog")]
public Blog Blog { get; set; }
}

public class TrailPost
{
[JsonProperty("id")]
public string Id { get; set; }
}
}

0 comments on commit a49353f

Please sign in to comment.