DataTable Lazy Loading problem with very high number of rows

UI Components for JSF
sunset88
Posts: 19
Joined: 26 Jul 2014, 11:24

22 Sep 2014, 09:49

Hi,
I'm using PrimeFaces 5 on Tomcat 7.

I'm using the p:dataTable component with Lazy Loading implementation and pagination feature.
I'm facing the following problem:

- if the number of rows in the resultSet is too high (for instance starting from 10 millions of rows), the query gets executed but the browser blocks (tested both on Chrome and Mozilla) and is not able to display the data-table data.

I did also the following tests:
- I set a fake small total count in the lazy loading implementation and this time the browser was able to show the result;
- I put fake high total count on those tables that have a limited number of rows and that were displayed without problems, and this time again the browser blocked and was not able to show the results anymore;

Here is the code:

- .xhtml

Code: Select all

<html xmlns="http://www.w3.org/1999/xhtml"
	xmlns:h="http://java.sun.com/jsf/html"
	xmlns:f="http://java.sun.com/jsf/core"
	xmlns:ui="http://java.sun.com/jsf/facelets"
	xmlns:c="http://java.sun.com/jsp/jstl/core"
	xmlns:p="http://primefaces.org/ui">



<h:head>
	<title>Web client</title>
</h:head>
<h:body>
	<h:form id="mainform">

		<p:dataTable widgetVar="dataTableWidgetVar" draggableColumns="true"
			scrollable="true" scrollWidth="100%" scrollHeight="300" var="row"
			value="#{queryResultBean.lazyModel}" paginator="true" id="table"
			sortMode="multiple"
			paginatorTemplate="Display {RowsPerPageDropdown} item(s) {FirstPageLink} {PreviousPageLink} {CurrentPageReport} {NextPageLink} {LastPageLink}    Go to page {JumpToPageDropdown}"
			currentPageReportTemplate="{startRecord}-{endRecord} out of {totalRecords} item(s)"
			paginatorPosition="both" rowsPerPageTemplate="20,50,100,300,500"
			rows="#{queryResultBean.selectionsToPost.getQueriesGeneralOptions().limit}"
			styleClass="data-table" lazy="true" rowIndexVar="rowIndex">

			<!--Start of Table Header -->
			<f:facet name="header">
				<h:outputText value="Result Table" />
			</f:facet>

			<!--End of Table Header -->

			<!-- Row values extraction -->
			<p:column headerText="#">
				<h:outputText value="#{row.number}" />
			</p:column>

			<c:forEach
				items="#{queryResultBean.rows.get(rowIndex % queryResultBean.selectionsToPost.getQueriesGeneralOptions().limit).attributesGroup}"
				var="attrGrpValues">

				<c:forEach items="#{attrGrpValues.attribute}" var="attrValues">
					<p:column style="width: -moz-max-content;"
						headerText="#{attrGrpValues.alias == null ? attrGrpValues.name : attrGrpValues.alias } #{attrValues.name}">

						<h:outputLink target="_blank" value="#{attrValues.link}"
							id="cell-element" disabled="#{attrValues.link == null}">
							<h:outputText value="#{attrValues.value}" />
						</h:outputLink>
					</p:column>

				</c:forEach>
			</c:forEach>
		</p:dataTable>

	</h:form>
</h:body>
</html>
The query result Bean:

Code: Select all



import javax.annotation.PostConstruct;
import javax.faces.bean.ManagedBean;
import javax.faces.bean.ViewScoped;
import javax.faces.context.ExternalContext;
import javax.faces.context.FacesContext;
import javax.faces.context.Flash;

import org.primefaces.model.LazyDataModel;
import org.primefaces.model.SortMeta;


@ManagedBean
@ViewScoped
public class QueryResultBean implements Serializable {

	private final static Logger logger = LoggerFactory
			.getLogger(QueryResultBean.class);

	private static final long serialVersionUID = 1L;

	private boolean aliasNeeded = false;

	private LazyDataModel<RowType> lazyModel;

	protected List<RowType> rows;

	protected FeatureDetails selectionsToPost = new FeatureDetails();

	private String limit;

	private int contatore = 1;

	@PostConstruct
	public void init() {

		// GET USER SELECTIONS
		FacesContext context = FacesContext.getCurrentInstance();
		ExternalContext extContext = context.getExternalContext();

		Flash flash = context.getExternalContext().getFlash();

		// EXTRACT QUERY GENERAL OPTIONS
		QueryGeneralOption genOpt = (QueryGeneralOption) flash.get("genOp");

		//set all user specified selections that will be posted to a service
		selectionsToPost.add(...);

		rows = null;

		lazyModel = new MyLazyDataModel(this);

	}
	

	public List<RowType> getRows() {
		if (rows == null) {
			List<SortMeta> multiSortMeta = new ArrayList<SortMeta>();
			Map<String, Object> filters = new HashMap<String, Object>();
			lazyModel.load(0, Integer.valueOf(selectionsToPost.getQueriesGeneralOptions().getLimit()), multiSortMeta, filters);
		}
		return rows;
	}


	public void setRows(List<RowType> rows) {
		this.rows = rows;
	}

	public LazyDataModel<RowType> getLazyModel() {
		return lazyModel;
	}

	public FeatureDetails getSelectionsToPost() {
		return selectionsToPost;
	}

}
The lazy model implementation:

Code: Select all


import org.primefaces.model.LazyDataModel;
import org.primefaces.model.SortMeta;

public class MyLazyDataModel extends LazyDataModel<RowType> {
	
	private static final long serialVersionUID = 1L;  

	private int totalRowCount = 0;   	   

	private List<SortMeta> multiSortMeta = new ArrayList<SortMeta>();

	//Map<String, Object> filters = new  HashMap<String, Object>();

	private List<RowType> data = null; 	

	private int first = 0;

	private QueryResultBean queryResultBean ;	
	

	public MyLazyDataModel(QueryResultBean queryResultBean) {
		super();
		this.queryResultBean = queryResultBean;
	}


	@Override
	public List<RowType> load(int first, int pageSize,
			List<SortMeta> multiSortMeta, Map<String, Object> filters) {

		FeatureDetails selectionsToPost = queryResultBean.selectionsToPost;
		
		data = new ArrayList<RowType>();

		Client4_RESTCalls client = new Client4_RESTCalls();		

		//SETTING PAGINATION OPTIONS
		selectionsToPost.getQueriesGeneralOptions().setLimit(String.valueOf(pageSize));		
		selectionsToPost.getQueriesGeneralOptions().setOffset(String.valueOf(first));
		
		//QUERY RESULT
		PublicDataDetails queryResult = client.getQueryResult(selectionsToPost);

		data = queryResult.getTableResult().get(0).getRowsGroup().getRow();
		
		int pageOffset = Integer.parseInt(selectionsToPost.getQueriesGeneralOptions().getOffset());


		if(!data.isEmpty()){
			if(queryResult.getTableResult().get(0).getEntriesTotalCount().getExactCount()!=null) {

				totalRowCount = Integer.valueOf(queryResult.getTableResult().get(0).getEntriesTotalCount().getExactCount());
			}else{

				totalRowCount= Integer.valueOf(queryResult.getTableResult().get(0).getEntriesTotalCount().getEstimatedCount());
				
			}

			multiSortMeta = new ArrayList<SortMeta>();

			filters = new  HashMap<String, Object>(); 

			this.setRowCount(totalRowCount);
			
			queryResultBean.rows = data;
		}
		
		return  data;
	}


	public List<RowType> getData() {
		return data;
	}

	public void setData(List<RowType> data) {
		this.data = data;
	}

}
Thank you in advance

User avatar
andyba
Expert Member
Posts: 2473
Joined: 31 Mar 2011, 16:27
Location: Steinfeld, near Bremen/Osnabrück, DE
Contact:

22 Sep 2014, 14:39

Where in code is it blocking? If it is in PrimeFaces code then there may be a PrimeFaces issue.
If it isn't then the problem is yours.
You can check this by simply debugging your load method.
PF 4.x (Elite versions), PF 5, Pf 5.1, PF 6.0
Glassfish 4.1, Mojarra 2.x, Java 8, Payara 4.1.1.
If you haven't read the forum rules read them now

sunset88
Posts: 19
Joined: 26 Jul 2014, 11:24

22 Sep 2014, 14:58

No, the code is not blocking. Everything is fine as regards the code and the query execution. It is the browser that blocks and is not able to show the results only when the number of rows is very high (tested both on Chrome and Mozilla).

Also to me seems like a PrimeFaces problem, because as I wrote above, if I put a fake small total row count, the browser is able to show the results.
So it seems like PrimeFaces has some display problems with lazy loading when the number of tuples is high, in my case starting from 10 millions of tuples.

How can I solve this? In my project the resultSet can often contain a very high number of tuples, also bigger than 10 millions.

User avatar
andyba
Expert Member
Posts: 2473
Joined: 31 Mar 2011, 16:27
Location: Steinfeld, near Bremen/Osnabrück, DE
Contact:

22 Sep 2014, 16:56

I have run LazyDataModel with millions of rows with no problems at all.

I have just checked your xhtml code and you are using c:forEach tags in an attempt to create dynamic columns.
Don't do this, use the dynamic columns support that PrimeFaces provides.

Apart from that use statically declared columns to test if PrimeFaces really does have a problem in the version you are using.
I am betting your c:forEach code is the root cause though.
PF 4.x (Elite versions), PF 5, Pf 5.1, PF 6.0
Glassfish 4.1, Mojarra 2.x, Java 8, Payara 4.1.1.
If you haven't read the forum rules read them now

sunset88
Posts: 19
Joined: 26 Jul 2014, 11:24

22 Sep 2014, 17:35

I've just tested with statically declared columns and removing the c:forEach but the problem remains.

smithh032772
Posts: 6144
Joined: 10 Sep 2011, 21:10

22 Sep 2014, 17:58

is it possible that this may be caused by the lack of DOCTYPE and <xml> tag above the <html> tag?

also,

Code: Select all

      <p:dataTable widgetVar="dataTableWidgetVar" draggableColumns="true"
         scrollable="true" scrollWidth="100%" scrollHeight="300" var="row"
         value="#{queryResultBean.lazyModel}" paginator="true" id="table"
         sortMode="multiple"
         paginatorTemplate="Display {RowsPerPageDropdown} item(s) {FirstPageLink} {PreviousPageLink} {CurrentPageReport} {NextPageLink} {LastPageLink}    Go to page {JumpToPageDropdown}"
         currentPageReportTemplate="{startRecord}-{endRecord} out of {totalRecords} item(s)"
         paginatorPosition="both" rowsPerPageTemplate="20,50,100,300,500"
         rows="#{queryResultBean.selectionsToPost.getQueriesGeneralOptions().limit}"
         styleClass="data-table" lazy="true" rowIndexVar="rowIndex">
1. draggableColumns=true may come with a price (or browser memory cost) in the client/browser for millions of rows

2. why var=row, id=table, rowIndexVar=rowIndex? you never know if these may be reserved words declared in primefaces.js file (or other primefaces javascript files).

3. is it possible that rows="..." and rowsPerPageTemplate="..." should have similar values? clarification: if rows="1 million", should rowsPerPageTemplate contain 1 million, too?
Howard

PrimeFaces 6.0, Extensions 6.0.0, Push (Atmosphere 2.4.0)
TomEE+ 1.7.4 (Tomcat 7.0.68), MyFaces Core 2.2.9, JDK8
JUEL 2.2.7 | OmniFaces | EclipseLink-JPA/Derby | Chrome

Java EE 6 Tutorial|NetBeans|Google|Stackoverflow|PrimeFaces|Apache

sunset88
Posts: 19
Joined: 26 Jul 2014, 11:24

22 Sep 2014, 22:08

Thank you for your suggestions. I tried them all, but the problem remains.

To me it seems like a PrimeFaces problem because:

1) If there were errors in my code logic or in the .xhtml code I would not have been able to see the results also when the result set is made up of a limited number of rows. Instead in case of small resultSet I have no problem. The browser manages to show the results;

2) I use the lazy loading feature and it works so I'm retrieving only a limited amount of rows per page (per request, usually 20). In fact the query is executed correctly using limit 20 and it retrieves only the limited amount of rows specified per page and not the whole result made up of millions of tuples;

3) in those cases where I have display problems I tried to put a fake small total count e.g.10000 ("20 out of 10000") and by doing so the browser was able to show the results. I did also the reverse test: for the tables that the browser was able to show I put a fake high total count, e.g. 10 millions and this time the browser was not able anymore to show the results.

I would like to post a screenshot to show you the browser block, but I cannot figure out how to do it. As it seems that the only way is using a URL.

Do you know how can I do it?

smithh032772
Posts: 6144
Joined: 10 Sep 2011, 21:10

22 Sep 2014, 22:52

upload your screen capture to a public website that stores image files, such as postimage.org, the website will provide a FORUM image URL to you, so then you reply to this topic, and click the 'Img' button after clicking Reply, and you put the URL between the 'img' tags. something like this below,

{img}URL{/img}
Howard

PrimeFaces 6.0, Extensions 6.0.0, Push (Atmosphere 2.4.0)
TomEE+ 1.7.4 (Tomcat 7.0.68), MyFaces Core 2.2.9, JDK8
JUEL 2.2.7 | OmniFaces | EclipseLink-JPA/Derby | Chrome

Java EE 6 Tutorial|NetBeans|Google|Stackoverflow|PrimeFaces|Apache

sunset88
Posts: 19
Joined: 26 Jul 2014, 11:24

23 Sep 2014, 09:38

Here is the browser block screenshot:

Image
upload
[/img]

And here is instead what I see in the normal case:

Image
caricare immagini
[/img]

smithh032772
Posts: 6144
Joined: 10 Sep 2011, 21:10

23 Sep 2014, 15:38

do you see any javascript errors in the browser console? please reply with the javascript errors, use the Code button, and copy/paste javascript errors between 'code' tags.

{code}...{/code}
Howard

PrimeFaces 6.0, Extensions 6.0.0, Push (Atmosphere 2.4.0)
TomEE+ 1.7.4 (Tomcat 7.0.68), MyFaces Core 2.2.9, JDK8
JUEL 2.2.7 | OmniFaces | EclipseLink-JPA/Derby | Chrome

Java EE 6 Tutorial|NetBeans|Google|Stackoverflow|PrimeFaces|Apache

Post Reply

Return to “PrimeFaces”

  • Information
  • Who is online

    Users browsing this forum: No registered users and 49 guests